{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b760a0f6",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network\n",
    "\n",
    "This example shows how to design a simple CNN.\n",
    "\n",
    "We also show the impact of quantization on accuracy and lastly how to run the model in FHE."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "253288cf",
   "metadata": {},
   "source": [
    "### Import required modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6200ab62",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.utils\n",
    "from concrete.compiler import check_gpu_available\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from tqdm import tqdm\n",
    "\n",
    "from concrete.ml.torch.compile import compile_torch_model\n",
    "\n",
    "# And some helpers for visualization.\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ed920ba",
   "metadata": {},
   "source": [
    "### Load the data-set and visualize it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9f955f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAAH5CAYAAABzvRxpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAykUlEQVR4nO3df3BU93nv8WdXQitkJOEELLGWAMtEVtANwoFqQwp1HMtD5JsMJrVLiJo4CrFD4psbW+HOreyMpc60KDO+deg4ICc3JWqbNEDT2sy9l6ixVZuGAKEFp0lwbBCWrVVkiR+1tRI2+rG7948Eyaoh4ln2nLP7nPdrZmdAfL/7fI8+u+dhtavvCSSTyaQAAAATgl4vAAAApA+NHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAIbluFkskEtLf3y+FhYUSCATcLI3LSCaTMjw8LOFwWILB9Pw/j5wzCxn7Q7pzJuPMc6UZu9rY+/v7pby83M2SuELRaFTKysrScl/knJnI2B/SlTMZZ66ZMna1sRcWFoqIyGq5Q3JllmN1zjXWquf89y/9g2r8nz9/h7rGkocG1XMmBk+r56juX8blgOybzCYd3Mo5FaVdc1Tjbyw4o67x/75xi3rO3O8dUc+5Un7L+M2PrVSN3/61b6hr/MVAnXrOwG0j6jka6c7ZrYxfbdWfr4998q9U438wMlddY3f9CvWcTDlfu9rYL/44J1dmSW7AuQdKTl6+ek7BnBzV+GCBvkZuME89Rxz8PomIyG83FE7nj9rcyjkVeXN0GeQX6NefyuPP0e+TzzLOnaX7/s8p1P/YOm9E/1x2/PuU5pzdyjiYr3++FCkzKxDd+V0ku8/XKb0Rs337dlm8eLHk5+dLJBKRI0ece7UBb5CxfWTsD+TsP+rGvnv3bmlqapKWlhY5duyY1NTUyNq1a+X0aWd/BAH3kLF9ZOwP5OxP6sb+2GOPyb333iuNjY2ydOlSeeKJJ6SgoEB27tzpxPrgATK2j4z9gZz9SdXYx8bG5OjRo1JXN/XBkWAwKHV1dXLo0KF3jB8dHZVYLDbthsymzViEnLMNGfsD52v/UjX2s2fPSjwel5KSkmlfLykpkYGBgXeMb2trk+Li4skbvzqR+bQZi5BztiFjf+B87V+O7jzX3NwsQ0NDk7doNOpkOXiEnO0jY/vI2A7Vr7vNmzdPcnJyZHBw+u9jDw4OSmlp6TvGh0IhCYVCV7dCuEqbsQg5Zxsy9gfO1/6lesWel5cnK1askK6ursmvJRIJ6erqklWrVqV9cXAfGdtHxv5Azv6l3qCmqalJ7rnnHlm5cqXU1tbKtm3b5Pz589LY2OjE+uABMraPjP2BnP1J3dg3bNggZ86ckUceeUQGBgZk+fLl0tnZ+Y4PaCB7kbF9ZOwP5OxPgWQymXSrWCwWk+LiYvmQrHN0i8LGl15Vz/lE4euq8R/42V3qGoeX/0A9Z0XrF1Tj533r0r+udDkTyXF5TvbK0NCQFBUVqeZejls5p2Ls6UWq8c9W71XXuKHzc+o5lZ/9N/WcK5XNGSduuVk95+nvf0c1/sT4eXWNF8b0jbH9PUvUczTSnXOqGZ9o1+393vbhv9cuTf5s50bV+F9+eYe6xu9/+fPqOXP+/qfqORpXmjHXYwcAwBAaOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMER9ERgvTHx4hWr8Jwp/pq5R/5FPqMYX//xFdY0/OnCbes5/3BxXjZ+nrpC9UtlH/JuV31DOuEZdo+gXeeo5uLSX79RfH3zr2ZtU4/+q61Z1jVMbnlDPaVfPyE5V7THV+L/9U93e8iIiX93/fdX4XcPXqms4ve+7k3jFDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGEJjBwDAEBo7AACG0NgBADCExg4AgCE0dgAADKGxAwBgCI0dAABDsuIiMBferVvmV0+/T10jkcJFXbT+9Rc3Ol4jm/W2flA1fm/jo+oalbP0F3XRuv5H59RzdJf68Y+bvvayes7uXt3Fln74gP5xdOvxT6rn5Mmr6jnZSH0uXValrvGJwtdV4//oZf0FuHJL9e1xYmBQPccJvGIHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGEJjBwDAEBo7AACG0NgBADAkO/aKv1b3/4/vHVqlrlEpR9RztHKLx9RzJobyHFhJZlrYelA1/oH29eoa+57/kXqO1vi8AvUcv/wPO6fkOtX4l/6kQl1j021d6jlas//4LfUcrgdwaalcp+O/vn+tavzNnf3qGtKpn/L8R8Kq8U7tLe+X8wkAAL5AYwcAwBAaOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMCQrLgKT/3pCNf733ndKXWNIOT63tERdY8PSo+o5e364Wj0H3jr9/tnqOaX7HVhIBvpV20LV+J6PPOHQSqbUPrRFPefawUMOrARXSnvxFO3FWUREzu0sVM8ZbHmXanzlF7gIDAAAmAGNHQAAQ1SNvbW1VQKBwLRbVVWVU2uDB8jYH8jZPjL2L/V77NXV1fLMM89M3UFuVrxNDwUy9gdyto+M/Umdcm5urpSWll7R2NHRURkdHZ38eywW05aDBzQZi5BztuK5bB8Z+5P6PfaTJ09KOByWiooKaWhokN7e3suObWtrk+Li4slbeXn5VS0W7tBkLELO2Yrnsn1k7E+qxh6JRKSjo0M6Ozulvb1denp6ZM2aNTI8PHzJ8c3NzTI0NDR5i0ajaVk0nKPNWIScsxHPZfvI2L9UP4qvr6+f/POyZcskEonIokWLZM+ePbJp06Z3jA+FQhIKha5+lXCNNmMRcs5GPJftI2P/uqpfd5s7d65UVlZKd3d3utaDDEPG/kDO9pGxf1xVYx8ZGZFTp07JggUL0rUeZBgy9gdyto+M/UPV2Lds2SL79++XV155RQ4ePCjr16+XnJwc2bhxo1Prg8vI2B/I2T4y9i/Ve+x9fX2yceNGOXfunMyfP19Wr14thw8flvnz5zu1PriMjP2BnO0jY/9SNfZdu3Y5tY7fqegl3SVaWsr+r7rGp+9rUo2fdecZdY1U3NDs7sUmvMoY7vIq5yV/HVeN37ryJnWNh+a9pBp/ZGu7usatDevUc85/T3chkms7ru65ny3P5RPtteo54X8OqMZfuFb/rvPfLH1MPefON76gnuME9ooHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGEJjBwDAEBo7AACG0NgBADBEtVe8VxI/f1E1fkP7V9Q1vvqV76vGbzt1m7rGvy7PUc/B5cUHT6vn3Hpct8f3s9V71TUmVuuubSAiIl/XT8lGwf3Pq8bvXzZbXePZWxpV4ye++h/6Gik8Lm74g8+pxl/boS6RlWa9oT8vfunPnN8H/86D+n3fKz75s/QvJAW8YgcAwBAaOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMMTVveKTyaSIiEzIuEjSuTrx0QvqOW+OxHU1zo+qa0wkx9VznDYhv1nTxWzSwa2cUzGhzC02nFDXiL+ZWY8Nv2WcmNA9/7WPCZHUHheJt5TrUj4m0p2zWxknLjh/vk5F4k39upw+x19pxoFkOp/tM+jr65Py8nK3ykEhGo1KWVlZWu6LnDMTGftDunIm48w1U8auNvZEIiH9/f1SWFgogUBg2r/FYjEpLy+XaDQqRUVFbi0pI3h57MlkUoaHhyUcDkswmJ53Zi6XMxmTsWVeH3u6c+Z8/U5eH/eVZuzqj+KDweCM/5MsKiry1QPl7bw69uLi4rTe30w5kzEZW+blsaczZ87Xl5fpGfPhOQAADKGxAwBgSMY09lAoJC0tLRIKhbxeiuv8cux+Oc5L8cux++U4L8VPx+6nY327bDluVz88BwAAnJUxr9gBAMDVo7EDAGAIjR0AAENo7AAAGJIRjX379u2yePFiyc/Pl0gkIkeOHPF6SY5rbW2VQCAw7VZVVeX1shxFzvZzJmMytijbMva8se/evVuampqkpaVFjh07JjU1NbJ27Vo5ffq010tzXHV1tbz22muTtwMHDni9JMeQs/2cyZiMLcuqjJMeq62tTd5///2Tf4/H48lwOJxsa2vzcFXOa2lpSdbU1Hi9DNeQs31kbB8ZZwdPX7GPjY3J0aNHpa6ubvJrwWBQ6urq5NChQx6uzB0nT56UcDgsFRUV0tDQIL29vV4vyRHkbD9nMiZj67IpY08b+9mzZyUej0tJScm0r5eUlMjAwIBHq3JHJBKRjo4O6ezslPb2dunp6ZE1a9bI8PCw10tLO3K2nzMZk7Fl2Zaxq1d3w5T6+vrJPy9btkwikYgsWrRI9uzZI5s2bfJwZUgncraPjO3Ltow9fcU+b948ycnJkcHBwWlfHxwclNLSUo9W5Y25c+dKZWWldHd3e72UtCPnKVZzJuMpZGxfpmfsaWPPy8uTFStWSFdX1+TXEomEdHV1yapVqzxcmftGRkbk1KlTsmDBAq+XknbkPMVqzmQ8hYzty/SMXf1RfCKRkP7+fiksLJRAICAiIps3b5bNmzdLdXW1rFixQnbs2CEjIyNy1113SSwWc3N5rnr44Yelvr5eysvLZWBgQLZu3SqBQEA++tGPunrcyWRShoeHJRwOSzCYnv/nkfOUTMiZjJ2VCRmLpD9nMp6SbRm7enW3vr4+KS8vd6scFKLRqJSVlaXlvsg5M5GxP6QrZzLOXDNl7Oor9sLCQhERWS13SK7McrP0jHKum68a/9b/zlfXyFsXVc9x2oSMywHZN5lNOriVc2nXHPWco6/pTlTX//Gv1DUyTTZnnArt4+LGgjPqGj9ZNVs9x2npztmtjKPNEfWc8eKEavynbvkXdY0t79a/f949fl41/k9u/Zhq/ERiTPaf/ZsZM06psW/fvl0effRRGRgYkJqaGnn88celtrZ2xnkXf5yTK7MkN5BZJ4OcYJ5qfO41IXWNTDtmERH57c9rLmZzUaoZv/2+nM45b44uMxGRnAJdbhmZmVYWZ5wK7eMiv0C//kw7ZhFJe85uZZyTr3+RFM/XNfb8Ofr1FxXq386YM66bk6vsOxf954z/M/XK/byloF+QsX1k7A/k7E/qxv7YY4/JvffeK42NjbJ06VJ54oknpKCgQHbu3OnE+uABMraPjP2BnP1J1di1WwqOjo5KLBabdkNmS2XbSHLOLmTsD5yv/UvV2LVbCra1tUlxcfHkjU9YZr5Uto0k5+xCxv7A+dq/HN2gprm5WYaGhiZv0WjmfSocV4+c7SNj+8jYDtWn4rVbCoZCIQmF9J8eh3dS2TaSnLMLGfsD52v/Ur1iZ0tB+8jYPjL2B3L2L/XvsTc1Nck999wjK1eulNraWtm2bZucP39eGhsbnVgfPEDG9pGxP5CzP6kb+4YNG+TMmTPyyCOPyMDAgCxfvlw6Ozvf8QENZC8yto+M/YGc/cnVveJjsZgUFxfLh2Rdxu3c1Nv6QdX4MeWWhiIiSx48rJ7jtInkuDwne2VoaEiKiorScp9u5fyFk/otH++8ZsSBlUz31Hn9Vrft71niwEp+I5szfv0z+h8ZH9narhp/4+7N6hp+eC67lbH23JuKef8eV8/J/eKlf0Pkd1lc+B+q8f0fGFaNv9KMPb1sKwAASC8aOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMER9EZhskFNynXrOpz7eNfOgt9n9ndvUNXKqb1LP0Yoff8nxGpnihbeuV8+58xrd9+fE+Hl1jYd/3qCes6jkjGp8fPC0ukY2urPpnx2vUfHUqOM1cHkLWw86XqP76x9Qz9lU8qJ6zoHbFyln6PaKv1K8YgcAwBAaOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAISYvAtPzhSXqOduKn1SN3//12eoav9q5Uj0nOKSLaMmD6hJZ6+nBKvWch+bpLgJTOesadY3EL4rVc+KDx9Vz/GDp7F+r52w9q7vYUnD/8+oauLw310dU4/v/IODQSqb88ON/4XgNEZHdn9RdHKz0685czIlX7AAAGEJjBwDAEBo7AACG0NgBADCExg4AgCE0dgAADKGxAwBgCI0dAABDaOwAABhCYwcAwBAaOwAAhmTFXvGvf2aVavyv7tuhrlF96D7V+DLR7+3d85Fvq+fUPPpF9Ry/yLv9VfWcNes/rxp/tiZHXSOVx997RZfzwtaD6hrZaGneoHrO3nM3q8b3tr5PXeOGvz+nnhM/rrtOQbYqPPGGavzCL15Q1/hm5d+p52hteqBJPaf0ycx4XvKKHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGEJjBwDAEBo7AACGZMVFYEJDCdX4E+Pn1TWOr/qeavzWn9+krpGK6/+uWzU+7tA6rCh48qeq8fMk4tBKpruwcMyVOtnmB0PvV8/5zsIfq8Zv/fhpdY2H7tNf0OX2jY2q8cH9z6trZALtxW7ybtfXqOy/RjW+9qEvqGtc++Qh9ZxMwSt2AAAMobEDAGCIqrG3trZKIBCYdquqqnJqbfAAGfsDOdtHxv6lfo+9urpannnmmak7yM2Kt+mhQMb+QM72kbE/qVPOzc2V0tJSJ9aCDEHG/kDO9pGxP6nfYz958qSEw2GpqKiQhoYG6e3tvezY0dFRicVi027IfJqMRcg5W/Fcto+M/UnV2CORiHR0dEhnZ6e0t7dLT0+PrFmzRoaHhy85vq2tTYqLiydv5eXlaVk0nKPNWIScsxHPZfvI2L9Ujb2+vl7uvvtuWbZsmaxdu1b27dsnb7zxhuzZs+eS45ubm2VoaGjyFo1G07JoOEebsQg5ZyOey/aRsX9d1Scp5s6dK5WVldLdfelNVEKhkIRCoaspAY/NlLEIOVvAc9k+MvaPq/o99pGRETl16pQsWLAgXetBhiFjfyBn+8jYP1SNfcuWLbJ//3555ZVX5ODBg7J+/XrJycmRjRs3OrU+uIyM/YGc7SNj/1L9KL6vr082btwo586dk/nz58vq1avl8OHDMn/+fKfWB5eRsT+Qs31k7F+qxr5r1y6n1vE7aS/c8aUnf19dI3HLzarx2//mG+oa1YfuU88pGzyunnM1vMo4Fa9/ZpV6jvaCQkv+5wvqGqko+z85rtS5KFty/tt/vE09R3uBlqcH9bux3VV8TD3n5Tt1718v2a8uMU22ZHxi50r9nPGfqMbP++EpdY1svqAWe8UDAGAIjR0AAENo7AAAGEJjBwDAEBo7AACG0NgBADCExg4AgCE0dgAADKGxAwBgCI0dAABDaOwAABhyVddjt2TW2TdV4ytnXaOu8a7vzlHPweWd+YNx9Zyej3zbgZVMV32oQT2nTHk9BL+4of3S1w7/nXMWfk41/p9u+0t1jc+f+KR6TsVTo+o5fnDvyh+r5/xxyxbV+GsHD6lrZDNesQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGOLqXvHJZFJERCZkXCTpZuWZJeO6fZxjwwl1jYnxC/o5Sf1+6Kr7l9/c/8Vs0sGtnBNv6b+fqeSmFX9Tvye4kzlnc8bJxJh6jvZxMZLKc/l8Cvu+T+jWFVQ+JtKds1sZXxjRP/bjY7rvpdPnUbdcacaBZDqf7TPo6+uT8vJyt8pBIRqNSllZWVrui5wzExn7Q7pyJuPMNVPGrjb2RCIh/f39UlhYKIFAYNq/xWIxKS8vl2g0KkVFRW4tKSN4eezJZFKGh4clHA5LMJied2YulzMZk7FlXh97unPmfP1OXh/3lWbs6o/ig8HgjP+TLCoq8tUD5e28Ovbi4uK03t9MOZMxGVvm5bGnM2fO15eX6Rnz4TkAAAyhsQMAYEjGNPZQKCQtLS0SCoW8Xorr/HLsfjnOS/HLsfvlOC/FT8fup2N9u2w5blc/PAcAAJyVMa/YAQDA1aOxAwBgCI0dAABDaOwAABhCYwcAwJCMaOzbt2+XxYsXS35+vkQiETly5IjXS3Jca2urBAKBabeqqiqvl+UocrafMxmTsUXZlrHnjX337t3S1NQkLS0tcuzYMampqZG1a9fK6dOnvV6a46qrq+W1116bvB04cMDrJTmGnO3nTMZkbFlWZZz0WG1tbfL++++f/Hs8Hk+Gw+FkW1ubh6tyXktLS7KmpsbrZbiGnO0jY/vIODt4+op9bGxMjh49KnV1dZNfCwaDUldXJ4cOHfJwZe44efKkhMNhqaiokIaGBunt7fV6SY4gZ/s5kzEZW5dNGXva2M+ePSvxeFxKSkqmfb2kpEQGBgY8WpU7IpGIdHR0SGdnp7S3t0tPT4+sWbNGhoeHvV5a2pGz/ZzJmIwty7aMXb1sK6bU19dP/nnZsmUSiURk0aJFsmfPHtm0aZOHK0M6kbN9ZGxftmXs6Sv2efPmSU5OjgwODk77+uDgoJSWlnq0Km/MnTtXKisrpbu72+ulpB05T7GaMxlPIWP7Mj1jV1+xJxIJ6e/vl8LCQgkEAiIisnz5cvnhD38oH/7whyfHPP3003LfffdJLBZzc3meGhkZke7ubrn77rtdPe5kMinDw8MSDoclGEzP//PI+fK8yJmM3WXluUzGl5fpGbt6dbe+vj4pLy93qxwUotGolJWVpeW+yDkzkbE/pCtnMs5cM2Xs6iv2wsJCERFZLXdIrsxyrE73jpvVc9pW/YNq/MP/tEFd4z2PvaKeEz99Rj1HY0LG5YDsm8wmHdzKeWyv/qSzcM7rqvEDt42oa2SabM4457r56jkvteqa2pO3tKtrfLn7bvWcvHVR9RyNdOfsVsZu2PT8y+o5L15YoJ5zeN1C1Xjt+f1KM3a1sV/8cU6uzJLcgHMPlODsfPWcgsIcXY18fY3cYJ56TsDB75OIiCQv1gmk7S7dyjlxTUg9J2+OLgMn1++aLM44J4XnjPb5P6dQ/2Pr3BQee44/ltKcs1sZu0F7fhcRyc/VH7P2HK8+v19hxim9EePHLQX9hoztI2N/IGf/UTd2P28p6BdkbB8Z+wM5+5O6sT/22GNy7733SmNjoyxdulSeeOIJKSgokJ07dzqxPniAjO0jY38gZ39SNXbtloKjo6MSi8Wm3ZDZUtk2kpyzCxn7A+dr/1I1du2Wgm1tbVJcXDx541cnMl8q20aSc3YhY3/gfO1fju4819zcLENDQ5O3aNTZX/eAN8jZPjK2j4ztUP26m3ZLwVAoJKGQ/tdC4J1Uto0k5+xCxv7A+dq/VK/Y8/LyZMWKFdLV1TX5tUQiIV1dXbJq1aq0Lw7uI2P7yNgfyNm/1BvUNDU1yT333CMrV66U2tpa2bZtm5w/f14aGxudWB88QMb2kbE/kLM/qRv7hg0b5MyZM/LII4/IwMCALF++XDo7O9/xAQ1kLzK2j4z9gZz9ydWLwMRiMSkuLpYPyTpHtygMH9bvlbzu3c87sJLp9p7T72Hf/4FhB1YyZSI5Ls/JXhkaGpKioqK03GeqOedU36Sqs+/p3dqluWLrWd1xiIjsXzbbgZX8RiZlrDX29CL1nGer96rG37h7s7rGX3z0u+o5LTs+rRpf+vWDqvHpztmtjFPx+md0byUc2aq/HkAq7rhddw2R+PGXVOOvNGNPr8cOAADSi8YOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMUV8EJhs894J+r+4jxQtV48v+8Li6xuOvdqrnbFrfpBpf8ORP1TUyxfi8AsdrNPauUY0/8mvd40JE5M+X6fYqFxHZL0vUc7KR9noAz1brrwdQfahBNX7Jg4fVNR4s/oR6jrxvTDX8nVdMx0UPPfy3jtfQnitE9Hu/O4VX7AAAGEJjBwDAEBo7AACG0NgBADCExg4AgCE0dgAADKGxAwBgCI0dAABDaOwAABhCYwcAwBAaOwAAhtDYAQAwxORFYJb8dVw95+nvf081vvGw/gIBL4yVqOcUnnhDNV5/5Jlj1ou/drzG4LrZqvG1e3vVNZbmDarniE8uAiNnX3e8xLu+O8fxGsEhk6fOtMgpuU41/tUd89U17rzmZ+o5fsIrdgAADKGxAwBgCI0dAABDaOwAABhCYwcAwBAaOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ0xueHzhXXmO1/jOwh+r59xx+wb1nPjxl9RzslV88LRq/NazN6lr7Hv+R6rxN3R+Tl2jeUGnek5Ote5YsvVxMfzBG7xeAhw2XnW9anzt9d3qGk+d110P4M5rRtQ1nntBf36plH9Tz3ECr9gBADCExg4AgCE0dgAADKGxAwBgCI0dAABDaOwAABhCYwcAwBAaOwAAhtDYAQAwhMYOAIAhNHYAAAyhsQMAYEhWXAQmccvNqvE/3v5NdY0bd29Wjc9fOKyu0fB9/QUCDmxcrhqfrRcHScX+ZbPVc569pVE1vnK/PrO1O7+snrN42xnV+Lzb1SUyQuHBHsdrjBbrXq8UllynrrHwv7ymnpP7Z+9Sz8lGwf3Pq8b3f0BfY+tnPqUaf+fWdnWNf7rtL9VzviS/r57jBF6xAwBgCI0dAABDVI29tbVVAoHAtFtVVZVTa4MHyNgfyNk+MvYv9Xvs1dXV8swzz0zdQW5WvE0PBTL2B3K2j4z9SZ1ybm6ulJaWXtHY0dFRGR0dnfx7LBbTloMHNBmLkHO24rlsHxn7k/o99pMnT0o4HJaKigppaGiQ3t7ey45ta2uT4uLiyVt5eflVLRbu0GQsQs7ZiueyfWTsT6rGHolEpKOjQzo7O6W9vV16enpkzZo1Mjx86V/9am5ulqGhoclbNBpNy6LhHG3GIuScjXgu20fG/qX6UXx9ff3kn5ctWyaRSEQWLVoke/bskU2bNr1jfCgUklAodPWrhGu0GYuQczbiuWwfGfvXVf2629y5c6WyslK6u7vTtR5kGDL2B3K2j4z946oa+8jIiJw6dUoWLFiQrvUgw5CxP5CzfWTsH6rGvmXLFtm/f7+88sorcvDgQVm/fr3k5OTIxo0bnVofXEbG/kDO9pGxf6neY+/r65ONGzfKuXPnZP78+bJ69Wo5fPiwzJ8/36n1wWVk7A/kbB8Z+5eqse/atcupdfxOs178tWr8ifHz6ho3fe1l1fjxquvVNR76vv4CLTd+7lbV+CUPqktM41XGbtFeoOLEzpXqGqlcPGLTA02q8XnyqrrG23mVc3zwtGp8Y+8adY3qzb9UjT/ysYXqGpLCr3iXKR97V8vyczk0lHC8xgtjJY7XcAp7xQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGKLaK94r2v2lP3/ik+oazz6/VzU+lf3obz2uX5d2D/u4ukL2SmUf9w8t1e3Xf0vBj9U17v/0f1PPKdj/U/UcPxhcN1s959UduoucNLzn39Q1nv3SB9VzkD6FB3tU47eevUld46F5+mt7fKvkOtV4bW+7UrxiBwDAEBo7AACG0NgBADCExg4AgCE0dgAADKGxAwBgCI0dAABDaOwAABhCYwcAwBAaOwAAhtDYAQAwxNW94pPJpIiITMi4SNK5OhPnR9VzYsMJ1fiRcd14kdTWNZEYU42PJ8d19y+/GX8xm3RwK+fEWxfUc8ZGdN/PCwnd91NEZGJCv66gMjeNbM44qXz8i4jE39Q9zy6MZH/GIunP2a2MU6F9XKSScSyUwjk+Q87XgWQ6n+0z6Ovrk/LycrfKQSEajUpZWVla7oucMxMZ+0O6cibjzDVTxq429kQiIf39/VJYWCiBQGDav8ViMSkvL5doNCpFRUVuLSkjeHnsyWRShoeHJRwOSzCYnndmLpczGZOxZV4fe7pz5nz9Tl4f95Vm7OqP4oPB4Iz/kywqKvLVA+XtvDr24uLitN7fTDmTMRlb5uWxpzNnzteXl+kZ8+E5AAAMobEDAGBIxjT2UCgkLS0tEgqFvF6K6/xy7H45zkvxy7H75TgvxU/H7qdjfbtsOW5XPzwHAACclTGv2AEAwNWjsQMAYAiNHQAAQ2jsAAAYkhGNffv27bJ48WLJz8+XSCQiR44c8XpJjmttbZVAIDDtVlVV5fWyHEXO9nMmYzK2KNsy9ryx7969W5qamqSlpUWOHTsmNTU1snbtWjl9+rTXS3NcdXW1vPbaa5O3AwcOeL0kx5Cz/ZzJmIwty6qMkx6rra1N3n///ZN/j8fjyXA4nGxra/NwVc5raWlJ1tTUeL0M15CzfWRsHxlnB09fsY+NjcnRo0elrq5u8mvBYFDq6urk0KFDHq7MHSdPnpRwOCwVFRXS0NAgvb29Xi/JEeRsP2cyJmPrsiljTxv72bNnJR6PS0lJybSvl5SUyMDAgEerckckEpGOjg7p7OyU9vZ26enpkTVr1sjw8LDXS0s7crafMxmTsWXZlrGrV3fDlPr6+sk/L1u2TCKRiCxatEj27NkjmzZt8nBlSCdyto+M7cu2jD19xT5v3jzJycmRwcHBaV8fHByU0tJSj1bljblz50plZaV0d3d7vZS0I+cpVnMm4ylkbF+mZ+xpY8/Ly5MVK1ZIV1fX5NcSiYR0dXXJqlWrPFyZ+0ZGRuTUqVOyYMECr5eSduQ8xWrOZDyFjO3L9Ixd/VF8IpGQ/v5+KSwslEAgICIimzdvls2bN0t1dbWsWLFCduzYISMjI3LXXXdJLBZzc3muevjhh6W+vl7Ky8tlYGBAtm7dKoFAQD760Y+6etzJZFKGh4clHA5LMJie/+eR85RMyJmMnZUJGYukP2cynpJtGbt6dbe+vj4pLy93qxwUotGolJWVpeW+yDkzkbE/pCtnMs5cM2Xs6iv2wsJCERFZLXdIrsxyrM4bDbXqOVWffUE1/szG2eoa8dNn1HOcNiHjckD2TWaTDqnmnPPe96jqvPSlOarxIiJP3tKuGv/i2HXqGs2H/lA9p+Lv4qrxwQM/v+KxmZSxG17+2u+pxv/Dxx5X1/iTWz+mnuP08z/dOaeasfb8qz33iojcWKD7Xm55tzvvhd+17uOq8fFfnVSNv9KMU2rs27dvl0cffVQGBgakpqZGHn/8camtnTnMiz/OyZVZkhtw7mSQk5evnpM3J081PjeoGy8iEnDwmFP225/XXMzmolQzfvt9aXPOyQld8VgRkeBsfc5zCnU/oiwYy1HXSGVdubnKxq55LGVQxm4I5uu+/9rHhEiGPv/TnHPKz2Pl+Vd77hURyS/QfS+LUsg4FbnKc5j6MXGZjP8z9dH6eUtBvyBj+8jYH8jZn9SN/bHHHpN7771XGhsbZenSpfLEE09IQUGB7Ny504n1wQNkbB8Z+wM5+5OqsWu3FBwdHZVYLDbthsyWyraR5JxdyNgfOF/7l6qxa7cUbGtrk+Li4skbn7DMfKlsG0nO2YWM/YHztX85+omC5uZmGRoamrxFo1Eny8Ej5GwfGdtHxnaoPhWv3VIwFApJKKT7lCC8lcq2keScXcjYHzhf+5fqFTtbCtpHxvaRsT+Qs3+pf4+9qalJ7rnnHlm5cqXU1tbKtm3b5Pz589LY2OjE+uABMraPjP2BnP1J3dg3bNggZ86ckUceeUQGBgZk+fLl0tnZ+Y4PaCB7kbF9ZOwP5OxPru4VH4vFpLi4WD4k6xzdrerxV3+invPCmO6B3vy9T6trLGw9qJ7jtInkuDwne2VoaEiKiorScp+p5vzm+oiqTv2fPqdcmchfdd2qGp8onlDX6PnIt9Vzbty9WTV+yYOHr3hsJmWslVOi39L3vgO659kLb12vrnHg9kXqOVrxQd0mMunOOdWMx57WfW+erd6rXZqcGD+vGl//j19R1wj/i741Fjz5U/UcjSvN2NPLtgIAgPSisQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAEPUF4HJBj8Yer96jnbv5w/s/YW6Rn+reoqvaPdZ3v/kbHWNOQ/q/i/7p1/cpa6h3cdaRKTiqVH1HD94dcd89ZyleYMzD3qb7Z+7W11j+5FvqOd8/sQnVePzbleXyAi9v1ygGv/U4jnqGn/5yjrV+Ju+9rK6hnav/kzCK3YAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGJIVF4HJqb5JNf57J/WHVTZ4XDV+3bu71TXaZYl6DtLrXXf82vEaD9zxWfWc4PHnHVhJ5ult/aBq/K9W7VDXeO+3/odq/A0v6p/LlbOuUc/RXhxlibyqrpGN7rxmRD+neq9q/FMH9BeaaX9P9p6vecUOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGBIVuwVHz/+kmr8okd0e8uLiMSV41PZ3/hbJdep58QHT6vn4PJmP5CvGr9036C6xlvbLqjn5N2unpKVLiwcc7zGpz7epRq/tMH56weIiLz73wOu1PHaTV97WTW+pveLDq1kyr//D/01B9odWIdbeMUOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAEOy4iIwWtqLxoiIDDz4QdX4E+M/Udfggi7e0z42Hrjjs+oa39y3Uz1n0/om1fiCJ3+qrpEJ3tvcqxpfXdygrvHkym+pxlfOukZd46nzc9Rzru04pJ6TjbTnudKv68+Lr39mlXqOVuKWm9Vzgvufd2AlerxiBwDAEBo7AACGqBp7a2urBAKBabeqqiqn1gYPkLE/kLN9ZOxf6vfYq6ur5Zlnnpm6g1yTb9P7Ghn7AznbR8b+pE45NzdXSktLnVgLMgQZ+wM520fG/qR+j/3kyZMSDoeloqJCGhoapLf38p9yHR0dlVgsNu2GzKfJWIScsxXPZfvI2J9UjT0SiUhHR4d0dnZKe3u79PT0yJo1a2R4ePiS49va2qS4uHjyVl5enpZFwznajEXIORvxXLaPjP1L1djr6+vl7rvvlmXLlsnatWtl37598sYbb8iePXsuOb65uVmGhoYmb9FoNC2LhnO0GYuQczbiuWwfGfvXVX2SYu7cuVJZWSnd3d2X/PdQKCShUOhqSsBjM2UsQs4W8Fy2j4z946p+j31kZEROnTolCxYsSNd6kGHI2B/I2T4y9g9VY9+yZYvs379fXnnlFTl48KCsX79ecnJyZOPGjU6tDy4jY38gZ/vI2L9UP4rv6+uTjRs3yrlz52T+/PmyevVqOXz4sMyfP9+p9cFlZOwP5GwfGfuXqrHv2rXLqXWk1YmdK9Vzej6yQ1djXF0ipXUFh3Qfg7jp26+rxifjoyK/mvq7lxnnlFynGn+2/kZ1jdFrA6rxGxq71DVSuahIbHGOanyBusJ0XuWsvUBI2R/qLxDyQMl61fh9z/9IXePhn69TzymT4+o5V8OrjN14Hn/3T/+XavzWs+9X18iUC7qkgr3iAQAwhMYOAIAhNHYAAAyhsQMAYAiNHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMuarrsWeqJX8dV8+5tVy393PvL/WXPrz3tmfVc06+qdt3uftflqrGT4xfmLZXvKfmXasaXr35lw4t5Orcely/j3jp1w86sBJ/enWH7iInJ8bPq2u867tz1HP8YviDN6jGa/d9F9Ffj+HAxuXqGiIvpTAnM/CKHQAAQ2jsAAAYQmMHAMAQGjsAAIbQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGEJjBwDAEFf3ik8mkyIiMiHjIknn6iQmLqjnTJwf1dW4oK9xYWRcPWfszTHV+Ilx3brivx1/MZt0SDXnZFyXwdiI7nuTigsJfWbax5KISDCpr3OlJuQ3950JGbsh/qbu+z8ynFDX0D7PREQmHMxYJP05p5qx9nuTyvc/Nks3Z0J5bhERiTucVyquNONAMp3P9hn09fVJeXm5W+WgEI1GpaysLC33Rc6ZiYz9IV05k3HmmiljVxt7IpGQ/v5+KSwslEAgMO3fYrGYlJeXSzQalaKiIreWlBG8PPZkMinDw8MSDoclGEzPOzOXy5mMydgyr4893Tlzvn4nr4/7SjN29UfxwWBwxv9JFhUV+eqB8nZeHXtxcXFa72+mnMmYjC3z8tjTmTPn68vL9Iz58BwAAIbQ2AEAMCRjGnsoFJKWlhYJhUJeL8V1fjl2vxznpfjl2P1ynJfip2P307G+XbYct6sfngMAAM7KmFfsAADg6tHYAQAwhMYOAIAhNHYAAAyhsQMAYEhGNPbt27fL4sWLJT8/XyKRiBw5csTrJTmutbVVAoHAtFtVVZXXy3IUOdvPmYzJ2KJsy9jzxr57925pamqSlpYWOXbsmNTU1MjatWvl9OnTXi/NcdXV1fLaa69N3g4cOOD1khxDzvZzJmMytiyrMk56rLa2Nnn//fdP/j0ejyfD4XCyra3Nw1U5r6WlJVlTU+P1MlxDzvaRsX1knB08fcU+NjYmR48elbq6usmvBYNBqaurk0OHDnm4MnecPHlSwuGwVFRUSENDg/T29nq9JEeQs/2cyZiMrcumjD1t7GfPnpV4PC4lJSXTvl5SUiIDAwMercodkUhEOjo6pLOzU9rb26Wnp0fWrFkjw8PDXi8t7cjZfs5kTMaWZVvGrl62FVPq6+sn/7xs2TKJRCKyaNEi2bNnj2zatMnDlSGdyNk+MrYv2zL29BX7vHnzJCcnRwYHB6d9fXBwUEpLSz1alTfmzp0rlZWV0t3d7fVS0o6cp1jNmYynkLF9mZ6xp409Ly9PVqxYIV1dXZNfSyQS0tXVJatWrfJwZe4bGRmRU6dOyYIFC7xeStqR8xSrOZPxFDK2L+Mz9vrTe7t27UqGQqFkR0dH8oUXXkjed999yblz5yYHBga8XpqjvvKVrySfe+65ZE9PT/InP/lJsq6uLjlv3rzk6dOnvV6aI8jZfs5kTMZWZVvGnjf2ZDKZfPzxx5MLFy5M5uXlJWtra5OHDx/2ekmO27BhQ3LBggXJvLy85PXXX5/csGFDsru72+tlOYqc7edMxmRsUbZlzPXYAQAwxPOd5wAAQPrQ2AEAMITGDgCAITR2AAAMobEDAGAIjR0AAENo7AAAGEJjBwDAEBo7AACG0NgBADCExg4AgCH/H/DzqsqCo/I4AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x600 with 16 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X, y = load_digits(return_X_y=True)\n",
    "\n",
    "# The sklearn Digits data-set, though it contains digit images, keeps these images in vectors\n",
    "# so we need to reshape them to 2D first. The images are 8x8 px in size and monochrome\n",
    "X = np.expand_dims(X.reshape((-1, 8, 8)), 1)\n",
    "\n",
    "nplot = 4\n",
    "fig, ax = plt.subplots(nplot, nplot, figsize=(6, 6))\n",
    "for i in range(0, nplot):\n",
    "    for j in range(0, nplot):\n",
    "        ax[i, j].imshow(X[i * nplot + j, ::].squeeze())\n",
    "plt.show()\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.25, shuffle=True, random_state=42\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c5e392d",
   "metadata": {},
   "source": [
    "### Define the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f43e2387",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TinyCNN(nn.Module):\n",
    "    \"\"\"A very small CNN to classify the sklearn digits data-set.\"\"\"\n",
    "\n",
    "    def __init__(self, n_classes) -> None:\n",
    "        \"\"\"Construct the CNN with a configurable number of classes.\"\"\"\n",
    "        super().__init__()\n",
    "\n",
    "        # This network has a total complexity of 1216 MAC\n",
    "        self.conv1 = nn.Conv2d(1, 8, 3, stride=1, padding=0)\n",
    "        self.conv2 = nn.Conv2d(8, 16, 3, stride=2, padding=0)\n",
    "        self.conv3 = nn.Conv2d(16, 32, 2, stride=1, padding=0)\n",
    "        self.fc1 = nn.Linear(32, n_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"Run inference on the tiny CNN, apply the decision layer on the reshaped conv output.\"\"\"\n",
    "        x = self.conv1(x)\n",
    "        x = torch.relu(x)\n",
    "        x = self.conv2(x)\n",
    "        x = torch.relu(x)\n",
    "        x = self.conv3(x)\n",
    "        x = torch.relu(x)\n",
    "        x = x.flatten(1)\n",
    "        x = self.fc1(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1449d54",
   "metadata": {},
   "source": [
    "### Train the CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f3035684",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   0%|          | 0/150 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   1%|          | 1/150 [00:00<00:19,  7.58it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   1%|▏         | 2/150 [00:00<00:18,  7.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   2%|▏         | 3/150 [00:00<00:18,  7.83it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   3%|▎         | 4/150 [00:00<00:18,  7.85it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   3%|▎         | 5/150 [00:00<00:18,  7.97it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   4%|▍         | 6/150 [00:00<00:17,  8.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   5%|▍         | 7/150 [00:00<00:17,  8.08it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   5%|▌         | 8/150 [00:00<00:17,  8.13it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   6%|▌         | 9/150 [00:01<00:17,  8.19it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   7%|▋         | 10/150 [00:01<00:17,  8.23it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   7%|▋         | 11/150 [00:01<00:16,  8.27it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   8%|▊         | 12/150 [00:01<00:16,  8.30it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   9%|▊         | 13/150 [00:01<00:16,  8.30it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:   9%|▉         | 14/150 [00:01<00:16,  8.31it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  10%|█         | 15/150 [00:01<00:16,  8.12it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  11%|█         | 16/150 [00:01<00:16,  8.17it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  11%|█▏        | 17/150 [00:02<00:16,  8.22it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  12%|█▏        | 18/150 [00:02<00:16,  8.24it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  13%|█▎        | 19/150 [00:02<00:16,  8.18it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  13%|█▎        | 20/150 [00:02<00:15,  8.21it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  14%|█▍        | 21/150 [00:02<00:15,  8.25it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  15%|█▍        | 22/150 [00:02<00:15,  8.03it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  15%|█▌        | 23/150 [00:02<00:16,  7.89it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  16%|█▌        | 24/150 [00:02<00:16,  7.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  17%|█▋        | 25/150 [00:03<00:15,  7.87it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  17%|█▋        | 26/150 [00:03<00:15,  7.85it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  18%|█▊        | 27/150 [00:03<00:15,  7.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  19%|█▊        | 28/150 [00:03<00:15,  7.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  19%|█▉        | 29/150 [00:03<00:15,  7.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  20%|██        | 30/150 [00:03<00:15,  7.87it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  21%|██        | 31/150 [00:03<00:15,  7.85it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  21%|██▏       | 32/150 [00:03<00:14,  7.89it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  22%|██▏       | 33/150 [00:04<00:14,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  23%|██▎       | 34/150 [00:04<00:14,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  23%|██▎       | 35/150 [00:04<00:14,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  24%|██▍       | 36/150 [00:04<00:14,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  25%|██▍       | 37/150 [00:04<00:14,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  25%|██▌       | 38/150 [00:04<00:14,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  26%|██▌       | 39/150 [00:04<00:13,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  27%|██▋       | 40/150 [00:04<00:13,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  27%|██▋       | 41/150 [00:05<00:13,  7.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  28%|██▊       | 42/150 [00:05<00:13,  7.89it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  29%|██▊       | 43/150 [00:05<00:13,  7.89it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  29%|██▉       | 44/150 [00:05<00:13,  7.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  30%|███       | 45/150 [00:05<00:13,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  31%|███       | 46/150 [00:05<00:13,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  31%|███▏      | 47/150 [00:05<00:13,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  32%|███▏      | 48/150 [00:06<00:12,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  33%|███▎      | 49/150 [00:06<00:12,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  33%|███▎      | 50/150 [00:06<00:12,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  34%|███▍      | 51/150 [00:06<00:12,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  35%|███▍      | 52/150 [00:06<00:12,  7.97it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  35%|███▌      | 53/150 [00:06<00:12,  7.99it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  36%|███▌      | 54/150 [00:06<00:12,  7.99it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  37%|███▋      | 55/150 [00:06<00:11,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  37%|███▋      | 56/150 [00:07<00:11,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  38%|███▊      | 57/150 [00:07<00:11,  7.86it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  39%|███▊      | 58/150 [00:07<00:11,  7.87it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  39%|███▉      | 59/150 [00:07<00:11,  7.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  40%|████      | 60/150 [00:07<00:11,  7.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  41%|████      | 61/150 [00:07<00:11,  7.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  41%|████▏     | 62/150 [00:07<00:11,  7.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  42%|████▏     | 63/150 [00:07<00:11,  7.75it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  43%|████▎     | 64/150 [00:08<00:11,  7.78it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  43%|████▎     | 65/150 [00:08<00:10,  7.81it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  44%|████▍     | 66/150 [00:08<00:10,  7.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  45%|████▍     | 67/150 [00:08<00:10,  7.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  45%|████▌     | 68/150 [00:08<00:10,  7.87it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  46%|████▌     | 69/150 [00:08<00:10,  7.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  47%|████▋     | 70/150 [00:08<00:10,  7.79it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  47%|████▋     | 71/150 [00:08<00:10,  7.73it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  48%|████▊     | 72/150 [00:09<00:10,  7.75it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  49%|████▊     | 73/150 [00:09<00:09,  7.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  49%|████▉     | 74/150 [00:09<00:09,  7.86it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  50%|█████     | 75/150 [00:09<00:09,  7.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  51%|█████     | 76/150 [00:09<00:09,  7.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  51%|█████▏    | 77/150 [00:09<00:09,  7.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  52%|█████▏    | 78/150 [00:09<00:09,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  53%|█████▎    | 79/150 [00:09<00:08,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  53%|█████▎    | 80/150 [00:10<00:08,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  54%|█████▍    | 81/150 [00:10<00:08,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  55%|█████▍    | 82/150 [00:10<00:08,  7.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  55%|█████▌    | 83/150 [00:10<00:08,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  56%|█████▌    | 84/150 [00:10<00:08,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  57%|█████▋    | 85/150 [00:10<00:08,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  57%|█████▋    | 86/150 [00:10<00:08,  7.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  58%|█████▊    | 87/150 [00:10<00:07,  7.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  59%|█████▊    | 88/150 [00:11<00:07,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  59%|█████▉    | 89/150 [00:11<00:07,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  60%|██████    | 90/150 [00:11<00:07,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  61%|██████    | 91/150 [00:11<00:07,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  61%|██████▏   | 92/150 [00:11<00:07,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  62%|██████▏   | 93/150 [00:11<00:07,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  63%|██████▎   | 94/150 [00:11<00:07,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  63%|██████▎   | 95/150 [00:11<00:06,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  64%|██████▍   | 96/150 [00:12<00:06,  7.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  65%|██████▍   | 97/150 [00:12<00:06,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  65%|██████▌   | 98/150 [00:12<00:06,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  66%|██████▌   | 99/150 [00:12<00:06,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  67%|██████▋   | 100/150 [00:12<00:06,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  67%|██████▋   | 101/150 [00:12<00:06,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  68%|██████▊   | 102/150 [00:12<00:06,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  69%|██████▊   | 103/150 [00:12<00:05,  7.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  69%|██████▉   | 104/150 [00:13<00:05,  7.89it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  70%|███████   | 105/150 [00:13<00:05,  7.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  71%|███████   | 106/150 [00:13<00:05,  7.89it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  71%|███████▏  | 107/150 [00:13<00:05,  7.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  72%|███████▏  | 108/150 [00:13<00:05,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  73%|███████▎  | 109/150 [00:13<00:05,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  73%|███████▎  | 110/150 [00:13<00:05,  7.86it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  74%|███████▍  | 111/150 [00:13<00:04,  8.00it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  75%|███████▍  | 112/150 [00:14<00:04,  8.02it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  75%|███████▌  | 113/150 [00:14<00:04,  8.02it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  76%|███████▌  | 114/150 [00:14<00:04,  8.01it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  77%|███████▋  | 115/150 [00:14<00:04,  8.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  77%|███████▋  | 116/150 [00:14<00:04,  8.00it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  78%|███████▊  | 117/150 [00:14<00:04,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  79%|███████▊  | 118/150 [00:14<00:04,  7.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  79%|███████▉  | 119/150 [00:14<00:03,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  80%|████████  | 120/150 [00:15<00:03,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  81%|████████  | 121/150 [00:15<00:03,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  81%|████████▏ | 122/150 [00:15<00:03,  7.98it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  82%|████████▏ | 123/150 [00:15<00:03,  7.99it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  83%|████████▎ | 124/150 [00:15<00:03,  7.97it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  83%|████████▎ | 125/150 [00:15<00:03,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  84%|████████▍ | 126/150 [00:15<00:03,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  85%|████████▍ | 127/150 [00:15<00:02,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  85%|████████▌ | 128/150 [00:16<00:02,  8.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  86%|████████▌ | 129/150 [00:16<00:02,  8.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  87%|████████▋ | 130/150 [00:16<00:02,  8.02it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  87%|████████▋ | 131/150 [00:16<00:02,  8.03it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  88%|████████▊ | 132/150 [00:16<00:02,  7.97it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  89%|████████▊ | 133/150 [00:16<00:02,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  89%|████████▉ | 134/150 [00:16<00:02,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  90%|█████████ | 135/150 [00:16<00:01,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  91%|█████████ | 136/150 [00:17<00:01,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  91%|█████████▏| 137/150 [00:17<00:01,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  92%|█████████▏| 138/150 [00:17<00:01,  7.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  93%|█████████▎| 139/150 [00:17<00:01,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  93%|█████████▎| 140/150 [00:17<00:01,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  94%|█████████▍| 141/150 [00:17<00:01,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  95%|█████████▍| 142/150 [00:17<00:01,  7.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  95%|█████████▌| 143/150 [00:17<00:00,  7.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  96%|█████████▌| 144/150 [00:18<00:00,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  97%|█████████▋| 145/150 [00:18<00:00,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  97%|█████████▋| 146/150 [00:18<00:00,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  98%|█████████▊| 147/150 [00:18<00:00,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  99%|█████████▊| 148/150 [00:18<00:00,  7.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training:  99%|█████████▉| 149/150 [00:18<00:00,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training: 100%|██████████| 150/150 [00:18<00:00,  7.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Training: 100%|██████████| 150/150 [00:18<00:00,  7.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAAGJCAYAAACZ7rtNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABQ6UlEQVR4nO3deVxU5f4H8M+ZhYEBhkVkUwRUckElFRc0y36ppKZppi1WLrfrNTU1WrXc2my5pqmlt+VqZaZZSrdywz1z33LfEnEFFIVhZ5g5vz+Q0ZFtDszMYZjP+/XiJXOWOd/5DtbHh2eeI4iiKIKIiIiIyAkp5C6AiIiIiKi6GGaJiIiIyGkxzBIRERGR02KYJSIiIiKnxTBLRERERE6LYZaIiIiInBbDLBERERE5LYZZIiIiInJaDLNERERE5LQYZomoSsOHD0dERES1zp0+fToEQbBtQU4gIiICw4cPl7sMq9jrPaqr7/358+chCAIWL15crfMFQcD06dNtWhORK2OYJXJigiBY9bVlyxa5S3V6S5cuxZw5c+Qug6zE94vIdQiiKIpyF0FE1bNkyRKLx99++y2SkpLw3XffWWzv2bMngoKCqn0dg8EAk8kEjUYj+dzi4mIUFxfD3d292tevDR555BEcPXoU58+ft+r4iIgIdO/evdqjd440ffp0zJgxA7b+34Gc773U90sKURRRWFgItVoNpVIp+fyCggKoVCqoVCqb10bkivg3iciJPfPMMxaPd+3ahaSkpDLb75aXlwetVmv1ddRqdbXqA8D/abug3NxceHp6Os17X1BQADc3NygU1v2yUhCEGgV0Z/+HHVFtw2kGRHVc9+7d0apVK+zfvx/3338/tFotJk+eDAD45Zdf0LdvX4SGhkKj0aBJkyZ45513YDQaLZ7j7jmzpXMG//3vf+OLL75AkyZNoNFo0KFDB+zdu9fi3PLmTQqCgHHjxiExMRGtWrWCRqNBdHQ01q5dW6b+LVu2IDY2Fu7u7mjSpAn+85//WD0X88yZMxg0aBCCg4Ph7u6Ohg0b4sknn0RWVpbFcUuWLEH79u3h4eEBf39/PPnkk7h48aJFD3///XekpKSYp25UZw7xuXPnMHjwYPj7+0Or1aJz5874/fffyxw3b948REdHQ6vVws/PD7GxsVi6dKl5f3Z2NiZOnIiIiAhoNBoEBgaiZ8+eOHDgQJU1bN++HR06dLDo590qmxN693zP0vfi+PHjePrpp+Hn54f77rvPYt/d59v7va/s/dqyZQsEQcCyZcvw1ltvoUGDBtBqtdDr9bhx4wZeeeUVtG7dGl5eXtDpdOjduzf++uuvKvszfPhweHl54fLlyxgwYAC8vLxQv359vPLKK2X+PlXUw7Nnz2L48OHw9fWFj48PRowYgby8PItz8/PzMX78eAQEBMDb2xv9+/fH5cuXOQ+XXFrt/yczEdVYRkYGevfujSeffBLPPPOMecrB4sWL4eXlhYSEBHh5eWHTpk2YOnUq9Ho9Pv744yqfd+nSpcjOzsa//vUvCIKAjz76CI899hjOnTtX5Wju9u3bsXLlSowZMwbe3t6YO3cuBg0ahAsXLqBevXoAgIMHD+Lhhx9GSEgIZsyYAaPRiLfffhv169evsraioiLEx8ejsLAQL774IoKDg3H58mX89ttvyMzMhI+PDwDgvffew5QpUzBkyBA8//zzuHbtGubNm4f7778fBw8ehK+vL958801kZWXh0qVLmD17NgDAy8uryhrulJaWhi5duiAvLw/jx49HvXr18M0336B///746aefMHDgQADAl19+ifHjx+Pxxx/HhAkTUFBQgMOHD2P37t14+umnAQCjR4/GTz/9hHHjxqFly5bIyMjA9u3bceLECbRr167CGo4cOYJevXqhfv36mD59OoqLizFt2rQaTUEpNXjwYERFReH999+vcrqCvd97a96vd955B25ubnjllVdQWFgINzc3HD9+HImJiRg8eDAiIyORlpaG//znP3jggQdw/PhxhIaGVnpdo9GI+Ph4dOrUCf/+97+xYcMGzJo1C02aNMELL7xQZd1DhgxBZGQkZs6ciQMHDuCrr75CYGAgPvzwQ/Mxw4cPx48//ohnn30WnTt3xtatW9G3b98qn5uoThOJqM4YO3asePdf6wceeEAEIC5cuLDM8Xl5eWW2/etf/xK1Wq1YUFBg3jZs2DAxPDzc/Dg5OVkEINarV0+8ceOGefsvv/wiAhB//fVX87Zp06aVqQmA6ObmJp49e9a87a+//hIBiPPmzTNv69evn6jVasXLly+bt505c0ZUqVRlnvNuBw8eFAGIK1asqPCY8+fPi0qlUnzvvfcsth85ckRUqVQW2/v27WvRg6qEh4eLw4YNMz+eOHGiCED8448/zNuys7PFyMhIMSIiQjQajaIoiuKjjz4qRkdHV/rcPj4+4tixY62updSAAQNEd3d3MSUlxbzt+PHjolKptOhn6fu7aNGiMs8BQJw2bZr5cen7+9RTT5U5Vq73XhQrfr82b94sAhAbN25c5ue/oKDA/D6USk5OFjUajfj2229bbLu7P8OGDRMBWBwniqLYtm1bsX379mV6UF4PR44caXHcwIEDxXr16pkf79+/XwQgTpw40eK44cOHl3lOIlfCaQZELkCj0WDEiBFltnt4eJi/z87OxvXr19GtWzfk5eXh5MmTVT7vE088AT8/P/Pjbt26ASj5dXpVevTogSZNmpgft2nTBjqdznyu0WjEhg0bMGDAAIsRsaZNm6J3795VPn/pyOu6devK/Kq21MqVK2EymTBkyBBcv37d/BUcHIyoqChs3ry5yutYa/Xq1ejYsaP5V/BAyWjhqFGjcP78eRw/fhwA4Ovri0uXLpWZrnEnX19f7N69G1euXLH6+kajEevWrcOAAQPQqFEj8/YWLVogPj6+Gq/I0ujRo60+1t7vvTWGDRtm8fMPlPw9KZ03azQakZGRAS8vLzRr1syqKRxA2T5069bNqr8PFZ2bkZEBvV4PAOapGGPGjLE47sUXX7Tq+YnqKoZZIhfQoEEDuLm5ldl+7NgxDBw4ED4+PtDpdKhfv775w2N3zystz52hCIA52N68eVPyuaXnl56bnp6O/Px8NG3atMxx5W27W2RkJBISEvDVV18hICAA8fHx+Oyzzyxe15kzZyCKIqKiolC/fn2LrxMnTiA9Pb3K61grJSUFzZo1K7O9RYsW5v0A8Prrr8PLywsdO3ZEVFQUxo4diz///NPinI8++ghHjx5FWFgYOnbsiOnTp1cZmK5du4b8/HxERUWV2VdeXVJFRkZafay933trlFevyWTC7NmzERUVBY1Gg4CAANSvXx+HDx+26u+Du7t7mWkQd76uqlT19yklJQUKhaJM7bbqCZGzYpglcgF3j0ABQGZmJh544AH89ddfePvtt/Hrr78iKSnJPD/PZDJV+bwVLUskWrHEU03OtdasWbNw+PBhTJ482fzBmejoaFy6dAlAyWsUBAFr165FUlJSma/yPhxlby1atMCpU6ewbNky3Hffffj5559x3333Ydq0aeZjhgwZgnPnzmHevHkIDQ3Fxx9/jOjoaKxZs8YmNVT0Aau7P8h0p/J+xiriiPe+KuXV+/777yMhIQH3338/lixZgnXr1iEpKQnR0dE1+vtgrdrQFyJnxA+AEbmoLVu2ICMjAytXrsT9999v3p6cnCxjVbcFBgbC3d0dZ8+eLbOvvG0Vad26NVq3bo233noLO3bsQNeuXbFw4UK8++67aNKkCURRRGRkJO65555Kn6emd7IKDw/HqVOnymwvnc4RHh5u3ubp6YknnngCTzzxBIqKivDYY4/hvffew6RJk8zLOoWEhGDMmDEYM2YM0tPT0a5dO7z33nsV/hq+fv368PDwwJkzZ8rsu7uu0hHBzMxMi+2lo8f2Zov3vjrv108//YQHH3wQX3/9tcX2zMxMBAQESH4+WwsPD4fJZEJycrLFCLuUvw9EdRFHZolcVOko0J2jPkVFRfj888/lKsmCUqlEjx49kJiYaDE39OzZs1aNQOr1ehQXF1tsa926NRQKBQoLCwEAjz32GJRKZbk3DBBFERkZGebHnp6eVv2quSJ9+vTBnj17sHPnTvO23NxcfPHFF4iIiEDLli0BwOKaAODm5oaWLVtCFEUYDAYYjcYydQQGBiI0NNT8usqjVCoRHx+PxMREXLhwwbz9xIkTWLduncWxOp0OAQEB2LZtm8V2R/1s1PS9B6r3fimVyjI/BytWrMDly5clPY+9lM5tvvt9mDdvnhzlENUaHJklclFdunSBn58fhg0bhvHjx0MQBHz33Xe16lea06dPx/r169G1a1e88MILMBqNmD9/Plq1aoVDhw5Veu6mTZswbtw4DB48GPfccw+Ki4vx3XffQalUYtCgQQCAJk2a4N1338WkSZNw/vx5DBgwAN7e3khOTsaqVaswatQovPLKKwCA9u3bY/ny5UhISECHDh3g5eWFfv36Wf1a3njjDfzwww/o3bs3xo8fD39/f3zzzTdITk7Gzz//bP7gUa9evRAcHIyuXbsiKCgIJ06cwPz589G3b194e3sjMzMTDRs2xOOPP46YmBh4eXlhw4YN2Lt3L2bNmlVpDTNmzMDatWvRrVs3jBkzBsXFxeY1bQ8fPmxx7PPPP48PPvgAzz//PGJjY7Ft2zacPn3a6tdbUzV574HqvV+PPPII3n77bYwYMQJdunTBkSNH8P3336Nx48Y2elU10759ewwaNAhz5sxBRkaGeWmu0velpr89IHJWDLNELqpevXr47bff8PLLL+Ott96Cn58fnnnmGTz00EM2+XS7LbRv3x5r1qzBK6+8gilTpiAsLAxvv/02Tpw4UeVqCzExMYiPj8evv/6Ky5cvQ6vVIiYmBmvWrEHnzp3Nx73xxhu45557MHv2bMyYMQMAEBYWhl69eqF///7m48aMGYNDhw5h0aJFmD17NsLDwyWF2aCgIOzYsQOvv/465s2bh4KCArRp0wa//vqrxTqh//rXv/D999/jk08+QU5ODho2bIjx48fjrbfeAgBotVqMGTMG69evN6/G0LRpU3z++edVrmXapk0brFu3DgkJCZg6dSoaNmyIGTNm4OrVq2XC7NSpU3Ht2jX89NNP+PHHH9G7d2+sWbMGgYGBVr/mmqjJew9U7/2aPHkycnNzsXTpUixfvhzt2rXD77//jjfeeMNWL6vGvv32WwQHB+OHH37AqlWr0KNHDyxfvhzNmjXjncXIZQlibRqGISKywoABA3Ds2LFy539S3cb3vqxDhw6hbdu2WLJkCYYOHSp3OUQOxzmzRFSr5efnWzw+c+YMVq9eje7du8tTEDkM3/uy7u4JAMyZMwcKhcLig5xEroTTDIioVmvcuDGGDx+Oxo0bIyUlBQsWLICbmxtee+01uUsjO+N7X9ZHH32E/fv348EHH4RKpcKaNWuwZs0ajBo1CmFhYXKXRyQLTjMgolptxIgR2Lx5M1JTU6HRaBAXF4f3338f7dq1k7s0sjO+92UlJSVhxowZOH78OHJyctCoUSM8++yzePPNN6FScXyKXBPDLBERERE5Lc6ZJSIiIiKnxTBLRERERE7L5SbYmEwmXLlyBd7e3lxgmoiIiKgWEkUR2dnZCA0NNd9UpiIuF2avXLnCT3wSEREROYGLFy+iYcOGlR7jcmHW29sbQElzdDqd3a9nMBiwfv169OrVC2q12u7Xc3bslzTsl/XYK2nYL2nYL+uxV9K4ar/0ej3CwsLMua0yLhdmS6cW6HQ6h4VZrVYLnU7nUj+E1cV+ScN+WY+9kob9kob9sh57JY2r98uaKaH8ABgREREROS2GWSIiIiJyWgyzREREROS0GGaJiIiIyGkxzBIRERGR02KYJSIiIiKnxTBLRERERE6LYZaIiIiInBbDLBERERE5LYZZOzuVmo1DGQLOpOXIXQoRERFRncMwa2ff7b6ARaeVWHc8Te5SiIiIiOochlk707mX3EdZX1AscyVEREREdQ/DrJ35eJSE2ax8g8yVEBEREdU9DLN2pvNQAQD0DLNERERENscwa2c+t6YZZDLMEhEREdkcw6yd6W5NM9Dnc84sERERka0xzNqZb+mc2QKOzBIRERHZGsOsnXHOLBEREZH9MMzaWelqBvkGE4qKTTJXQ0RERFS3MMzambdGBQEiAC7PRURERGRrDLN2plAIcFeWfM8wS0RERGRbDLMOoC2ZNsswS0RERGRjDLMOcOszYPwQGBEREZGNMcw6gFZVMmc2M79I5kqIiIiI6haGWQfQls6ZzePILBEREZEtMcw6wO05s7wLGBEREZEtMcw6gAc/AEZERERkFwyzDlA6Z5ZhloiIiMi2GGYdgEtzEREREdmHrGF25syZ6NChA7y9vREYGIgBAwbg1KlTVZ63YsUKNG/eHO7u7mjdujVWr17tgGqrj0tzEREREdmHrGF269atGDt2LHbt2oWkpCQYDAb06tULubm5FZ6zY8cOPPXUU/jHP/6BgwcPYsCAARgwYACOHj3qwMql0fIOYERERER2oZLz4mvXrrV4vHjxYgQGBmL//v24//77yz3n008/xcMPP4xXX30VAPDOO+8gKSkJ8+fPx8KFC+1ec3VwziwRERGRfcgaZu+WlZUFAPD396/wmJ07dyIhIcFiW3x8PBITE8s9vrCwEIWFhebHer0eAGAwGGAw2D9cGgyGO1YzKHLINZ1ZaX/YJ+uwX9Zjr6Rhv6Rhv6zHXknjqv2S8noFURRFO9ZiNZPJhP79+yMzMxPbt2+v8Dg3Nzd88803eOqpp8zbPv/8c8yYMQNpaWlljp8+fTpmzJhRZvvSpUuh1WptU3wV8oqBSXtLEu2sTsVQ8WN3RERERBXKy8vD008/jaysLOh0ukqPrTUjs2PHjsXRo0crDbLVMWnSJIuRXL1ej7CwMPTq1avK5tiCwWDAuvVJEACIAOIeeAj1vTV2v66zMhgMSEpKQs+ePaFWq+Uup9Zjv6zHXknDfknDflmPvZLGVftV+pt0a9SKMDtu3Dj89ttv2LZtGxo2bFjpscHBwWVGYNPS0hAcHFzu8RqNBhpN2fCoVqsd9kOhEACdhwpZ+cXIKxZd6oexuhz5/tQF7Jf12Ctp2C9p2C/rsVfSuFq/pLxWWX/hLYoixo0bh1WrVmHTpk2IjIys8py4uDhs3LjRYltSUhLi4uLsVaZN6NxL3hR+CIyIiIjIdmQNs2PHjsWSJUuwdOlSeHt7IzU1FampqcjPzzcf89xzz2HSpEnmxxMmTMDatWsxa9YsnDx5EtOnT8e+ffswbtw4OV6C1Xw8GGaJiIiIbE3WMLtgwQJkZWWhe/fuCAkJMX8tX77cfMyFCxdw9epV8+MuXbpg6dKl+OKLLxATE4OffvoJiYmJaNWqlRwvwWq6W0saMMwSERER2Y6sc2atWUhhy5YtZbYNHjwYgwcPtkNF9uNTOs0gj2GWiIiIyFa4SJSD6MzTDIplroSIiIio7mCYdRAfTjMgIiIisjmGWQcpXc0gM79I5kqIiIiI6g6GWQfx1ZaEWT1HZomIiIhshmHWQXTunGZAREREZGsMsw7CdWaJiIiIbI9h1kEYZomIiIhsj2HWQXjTBCIiIiLbY5h1kNKbJhQYTCgsNspcDREREVHdwDDrIF4aFQSh5HuOzhIRERHZBsOsgygUgnmtWS7PRURERGQbDLMOVLrWbGYewywRERGRLTDMOhBXNCAiIiKyLYZZB2KYJSIiIrIthlkH0jHMEhEREdkUw6wDcWSWiIiIyLYYZh2IYZaIiIjIthhmHYhhloiIiMi2GGYdqDTMcp1ZIiIiIttgmHUgjswSERER2RbDrAP5MswSERER2RTDrAOVLs3FO4ARERER2QbDrANxmgERERGRbTHMOpCPtiTMFhabUGAwylwNERERkfNjmHUgLzcVFELJ91zRgIiIiKjmGGYdSKEQeEtbIiIiIhtimHUwzpslIiIish2GWQfTud+6cUIBwywRERFRTTHMOpinRgkAyCnkB8CIiIiIaoph1sG8NCoAQF5hscyVEBERETk/hlkH87wVZnMYZomIiIhqjGHWwbRuJWE2l9MMiIiIiGqMYdbBvG7Nmc0r4sgsERERUU0xzDoYpxkQERER2Q7DrIN5mqcZMMwSERER1RTDrIOVjszmFnHOLBEREVFNMcw6WOk6sxyZJSIiIqo5hlkHM08z4MgsERERUY0xzDqYeZoBR2aJiIiIaoxh1sE4zYCIiIjIdhhmHYwjs0RERES2wzDrYF53rGYgiqLM1RARERE5N4ZZB9O6lUwzMJpEFBabZK6GiIiIyLkxzDpY6WoGAKcaEBEREdUUw6yDKRSCeXQ2t5DLcxERERHVBMOsDLS3RmdzODJLREREVCMMszLwurU8V14RwywRERFRTUgOs/n5+cjLyzM/TklJwZw5c7B+/XqbFlaXlS7PxZFZIiIiopqRHGYfffRRfPvttwCAzMxMdOrUCbNmzcKjjz6KBQsW2LzAush8S1vOmSUiIiKqEclh9sCBA+jWrRsA4KeffkJQUBBSUlLw7bffYu7cuTYvsC4y3wWM0wyIiIiIakRymM3Ly4O3tzcAYP369XjsscegUCjQuXNnpKSk2LzAukjLu4ARERER2YTkMNu0aVMkJibi4sWLWLduHXr16gUASE9Ph06ns3mBdZGXG8MsERERkS1IDrNTp07FK6+8goiICHTq1AlxcXEASkZp27Zta/MC6yLPO25pS0RERETVp6r6EEuPP/447rvvPly9ehUxMTHm7Q899BAGDhxo0+LqKvOcWY7MEhEREdVItdaZDQ4ORtu2baFQKKDX65GYmAhvb280b95c0vNs27YN/fr1Q2hoKARBQGJiYqXHb9myBYIglPlKTU2tzsuQDZfmIiIiIrINyWF2yJAhmD9/PoCSNWdjY2MxZMgQtGnTBj///LOk58rNzUVMTAw+++wzSeedOnUKV69eNX8FBgZKOl9upWE2j0tzEREREdWI5GkG27Ztw5tvvgkAWLVqFURRRGZmJr755hu8++67GDRokNXP1bt3b/Tu3VtqCQgMDISvr69VxxYWFqKwsND8WK/XAwAMBgMMBoPka0tVeo07r+VeMssA2QWOqcGZlNcvqhj7ZT32Shr2Sxr2y3rslTSu2i8pr1cQRVGU8uQeHh44ffo0wsLC8NxzzyE0NBQffPABLly4gJYtWyInJ0dywQAgCAJWrVqFAQMGVHjMli1b8OCDDyI8PByFhYVo1aoVpk+fjq5du1Z4zvTp0zFjxowy25cuXQqtVlutWmvq8A0BX59SIsJLxEutOTpLREREdKe8vDw8/fTTyMrKqnK1LMkjs2FhYdi5cyf8/f2xdu1aLFu2DABw8+ZNuLu7V69iK4WEhGDhwoWIjY1FYWEhvvrqK3Tv3h27d+9Gu3btyj1n0qRJSEhIMD/W6/UICwtDr169HLKUmMFgQFJSEnr27Am1Wg0A8P07A1+f2g83rTf69Oli9xqcSXn9ooqxX9Zjr6Rhv6Rhv6zHXknjqv0q/U26NSSH2YkTJ2Lo0KHw8vJCeHg4unfvDqBk+kHr1q2lPp0kzZo1Q7NmzcyPu3Tpgr///huzZ8/Gd999V+45Go0GGo2mzHa1Wu3QH4o7r6fTltSTW2R0qR9MKRz9/jg79st67JU07Jc07Jf12CtpXK1fUl6r5DA7ZswYdOzYERcvXkTPnj2hUJR8hqxx48Z49913pT5djXXs2BHbt293+HVrwqv0A2C8nS0RERFRjUgOswAQGxuL2NhYiKIIURQhCAL69u1r69qscujQIYSEhMhy7eq6fTtbzpclIiIiqolqrTP77bffonXr1vDw8ICHhwfatGlT4a/5K5OTk4NDhw7h0KFDAIDk5GQcOnQIFy5cAFAy3/W5554zHz9nzhz88ssvOHv2LI4ePYqJEydi06ZNGDt2bHVehmxKb2dbZDShqNgkczVEREREzkvyyOwnn3yCKVOmYNy4ceZVBLZv347Ro0fj+vXreOmll6x+rn379uHBBx80Py79oNawYcOwePFiXL161RxsAaCoqAgvv/wyLl++DK1WizZt2mDDhg0Wz+EMtLfuAAaUTDVwU7nJWA0RERGR85IcZufNm4cFCxZYjJj2798f0dHRmD59uqQw2717d1S2MtjixYstHr/22mt47bXXpJZc66iVCripFCgqNiGnsBi+WoZZIiIiouqQPM3g6tWr6NKl7HJSXbp0wdWrV21SlCvw4rxZIiIiohqTHGabNm2KH3/8scz25cuXIyoqyiZFuQLPW1MNcrmiAREREVG1SZ5mMGPGDDzxxBPYtm2bec7sn3/+iY0bN5Ybcql8nm6lI7MMs0RERETVJXlkdtCgQdi9ezcCAgKQmJiIxMREBAQEYM+ePRg4cKA9aqyTPDUMs0REREQ1Va11Ztu3b48lS5ZYbEtPT8f777+PyZMn26Swus6Tc2aJiIiIaqxa68yW5+rVq5gyZYqtnq7O83TjnFkiIiKimrJZmCVpSkdmczjNgIiIiKjaGGZlUjoym8dpBkRERETVxjArE47MEhEREdWc1R8AK73VbEWuXbtW42JcCVczICIiIqo5q8PswYMHqzzm/vvvr1ExrsQ8zaCI0wyIiIiIqsvqMLt582Z71uFyOM2AiIiIqOY4Z1YmXrfCbB6X5iIiIiKqNoZZmWjNI7OcZkBERERUXQyzMvHS3LppAqcZEBEREVUbw6xMPDnNgIiIiKjGGGZl4unGD4ARERER1ZTkMBsREYG3334bFy5csEc9LqN0ZLbAYEKx0SRzNURERETOSXKYnThxIlauXInGjRujZ8+eWLZsGQoLC+1RW52mvbXOLADkGfghMCIiIqLqqFaYPXToEPbs2YMWLVrgxRdfREhICMaNG4cDBw7Yo8Y6SaNSQKUQAPBDYERERETVVe05s+3atcPcuXNx5coVTJs2DV999RU6dOiAe++9F//9738hiqIt66xzBEHgLW2JiIiIasjqO4DdzWAwYNWqVVi0aBGSkpLQuXNn/OMf/8ClS5cwefJkbNiwAUuXLrVlrXWOp5sSWfkG5HKtWSIiIqJqkRxmDxw4gEWLFuGHH36AQqHAc889h9mzZ6N58+bmYwYOHIgOHTrYtNC6iCOzRERERDUjOcx26NABPXv2xIIFCzBgwACo1eoyx0RGRuLJJ5+0SYF1maeGy3MRERER1YTkMHvu3DmEh4dXeoynpycWLVpU7aJcheetu4DlFXGaAREREVF1SA6zpUF23759OHHiBACgRYsWiI2NtW1lLoA3TiAiIiKqGclh9tKlS3jqqafw559/wtfXFwCQmZmJLl26YNmyZWjYsKGta6yzvDhnloiIiKhGJC/N9fzzz8NgMODEiRO4ceMGbty4gRMnTsBkMuH555+3R411lvbWNINcTjMgIiIiqhbJI7Nbt27Fjh070KxZM/O2Zs2aYd68eejWrZtNi6vruJoBERERUc1IHpkNCwuDwWAos91oNCI0NNQmRbmK0jmzeUUMs0RERETVITnMfvzxx3jxxRexb98+87Z9+/ZhwoQJ+Pe//23T4uq620tzcZoBERERUXVInmYwfPhw5OXloVOnTlCpSk4vLi6GSqXCyJEjMXLkSPOxN27csF2ldZBX6ZxZTjMgIiIiqhbJYXbOnDl2KMM1ad04Z5aIiIioJiSH2WHDhtmjDpdkXpqLc2aJiIiIqkVymAVKPuyVmJhovmlCdHQ0+vfvD6VSadPi6rrbqxlwziwRERFRdUgOs2fPnkWfPn1w+fJl8/JcM2fORFhYGH7//Xc0adLE5kXWVVo3zpklIiIiqgnJqxmMHz8eTZo0wcWLF3HgwAEcOHAAFy5cQGRkJMaPH2+PGuss3gGMiIiIqGaqddOEXbt2wd/f37ytXr16+OCDD9C1a1ebFlfXmacZFBlhMolQKASZKyIiIiJyLpJHZjUaDbKzs8tsz8nJgZubm02KchWemttzjPMMnDdLREREJJXkMPvII49g1KhR2L17N0RRhCiK2LVrF0aPHo3+/fvbo8Y6y0OthPLWaGx2Qdm7qhERERFR5SSH2blz56JJkyaIi4uDu7s73N3d0bVrVzRt2hSffvqpPWqsswRBgI+HGgCQlc8wS0RERCSVpDmzoihCr9dj2bJluHz5snlprhYtWqBp06Z2KbCu8/FQ40ZuEbLyGGaJiIiIpJIcZps2bYpjx44hKiqKAdYGdLdGZvUFXNGAiIiISCpJ0wwUCgWioqKQkZFhr3pcjs695N8TnGZAREREJJ3kObMffPABXn31VRw9etQe9bgczpklIiIiqj7J68w+99xzyMvLQ0xMDNzc3ODh4WGx/8aNGzYrzhUwzBIRERFVn+QwO3v2bAgCF/e3ldIwq2eYJSIiIpJMcpgdPny4HcpwXRyZJSIiIqo+yXNmlUol0tPTy2zPyMiAUqks5wyqDMMsERERUfVJDrOiKJa7vbCwkLezrQZOMyAiIiKqPqunGcydOxdAyV2rvvrqK3h5eZn3GY1GbNu2Dc2bN7d9hXUcR2aJiIiIqs/qMDt79mwAJSOzCxcutJhS4ObmhoiICCxcuND2FdZxOoZZIiIiomqzOswmJycDAB588EGsXLkSfn5+divKlXBkloiIiKj6JM+Z3bx5s82C7LZt29CvXz+EhoZCEAQkJiZWec6WLVvQrl07aDQaNG3aFIsXL7ZJLXIpHZktLDahwGCUuRoiIiIi5yJ5aS6j0YjFixdj48aNSE9Ph8lksti/adMmq58rNzcXMTExGDlyJB577LEqj09OTkbfvn0xevRofP/999i4cSOef/55hISEID4+XupLqRW8NSoIAiCKJR8Cc1dzRQgiIiIia0kOsxMmTMDixYvRt29ftGrVqkY3UOjduzd69+5t9fELFy5EZGQkZs2aBQBo0aIFtm/fjtmzZzttmFUoBOjc1cjKNyAr34BAnbvcJRERERE5DclhdtmyZfjxxx/Rp08fe9RTqZ07d6JHjx4W2+Lj4zFx4sQKzyksLERhYaH5sV6vBwAYDAYYDPafp1p6jcqupXNXISvfgIzsfET4u3aYtaZfdBv7ZT32Shr2Sxr2y3rslTSu2i8pr1dymHVzc0PTpk2lnmYTqampCAoKstgWFBQEvV6P/Px8eHh4lDln5syZmDFjRpnt69evh1artVutd0tKSqp4Z5ESgIBN23ch7Vj56/i6mkr7RWWwX9Zjr6Rhv6Rhv6zHXknjav3Ky8uz+ljJYfbll1/Gp59+ivnz59doioGjTJo0CQkJCebHer0eYWFh6NWrF3Q6nd2vbzAYkJSUhJ49e0KtVpd7zPL0fbj49w3cEx2DPveG2r2m2syaftFt7Jf12Ctp2C9p2C/rsVfSuGq/Sn+Tbg3JYXb79u3YvHkz1qxZg+jo6DKNXblypdSntFpwcDDS0tIstqWlpUGn05U7KgsAGo0GGo2mzHa1Wu3QH4rKruerLblzWk6RyaV+UCvj6PfH2bFf1mOvpGG/pGG/rMdeSeNq/ZLyWiWHWV9fXwwcOFDqaTYRFxeH1atXW2xLSkpCXFycLPXYyu21ZotlroSIiIjIuUgOs4sWLbLZxXNycnD27Fnz4+TkZBw6dAj+/v5o1KgRJk2ahMuXL+Pbb78FAIwePRrz58/Ha6+9hpEjR2LTpk348ccf8fvvv9usJjnwLmBERERE1WP1TRPS09Mr3V9cXIw9e/ZIuvi+ffvQtm1btG3bFgCQkJCAtm3bYurUqQCAq1ev4sKFC+bjIyMj8fvvvyMpKQkxMTGYNWsWvvrqK6ddlqsU7wJGREREVD1Wj8yGhITg6tWrCAwMBAC0bt0aq1evRlhYGAAgIyMDcXFxMBqtv4tV9+7dIYoVf3q/vLt7de/eHQcPHrT6Gs6AYZaIiIioeqwemb07dJ4/f77MGmCVBVOqWGmY1RcwzBIRERFJYXWYtYYzLNVVG5nDLEdmiYiIiCSxaZil6tG5c5oBERERUXVYPWdWEARkZ2fD3d0doihCEATk5OSYF7WVsrgtWeKcWSIiIqLqsTrMiqKIe+65x+Jx6SoEpY85zaB6SsNsXpERBqMJaiUHzImIiIisYXWY3bx5sz3rcGml68wCJaOzAV5l71hGRERERGVZHWYfeOABe9bh0pQKAd4aFbILixlmiYiIiCTg77NrCd4FjIiIiEg6htlagstzEREREUnHMFtLcEUDIiIiIukYZmsJnUfJ9GWOzBIRERFZr8ZhVq/XIzExESdOnLBFPS6LI7NERERE0kkOs0OGDMH8+fMBAPn5+YiNjcWQIUPQpk0b/PzzzzYv0FUwzBIRERFJJznMbtu2Dd26dQMArFq1CqIoIjMzE3PnzsW7775r8wJdBcMsERERkXSSw2xWVhb8/f0BAGvXrsWgQYOg1WrRt29fnDlzxuYFugqGWSIiIiLpJIfZsLAw7Ny5E7m5uVi7di169eoFALh58ybc3d1tXqCr0JmX5iqWuRIiIiIi52H1HcBKTZw4EUOHDoWXlxfCw8PRvXt3ACXTD1q3bm3r+lwGR2aJiIiIpJMcZseMGYOOHTvi4sWL6NmzJxSKksHdxo0bc85sDTDMEhEREUknOcwCQGxsLGJjYwEARqMRR44cQZcuXeDn52fT4lyJjncAIyIiIpJM8pzZiRMn4uuvvwZQEmQfeOABtGvXDmFhYdiyZYut63MZpSOz2YXFMJpEmashIiIicg6Sw+xPP/2EmJgYAMCvv/6K5ORknDx5Ei+99BLefPNNmxfoKkrDLMDRWSIiIiJrSQ6z169fR3BwMABg9erVGDx4MO655x6MHDkSR44csXmBrkKtVEDrpgTAebNERERE1pIcZoOCgnD8+HEYjUasXbsWPXv2BADk5eVBqVTavEBXwg+BEREREUkj+QNgI0aMwJAhQxASEgJBENCjRw8AwO7du9G8eXObF+hKfDzUuJpVAH0BwywRERGRNSSH2enTp6NVq1a4ePEiBg8eDI1GAwBQKpV44403bF6gK9FxZJaIiIhIkmotzfX444+X2TZs2LAaF+PqOM2AiIiISBrJc2YBYOvWrejXrx+aNm2Kpk2bon///vjjjz9sXZvL0bkzzBIRERFJITnMLlmyBD169IBWq8X48eMxfvx4eHh44KGHHsLSpUvtUaPL4MgsERERkTSSpxm89957+Oijj/DSSy+Zt40fPx6ffPIJ3nnnHTz99NM2LdCV+PAuYERERESSSB6ZPXfuHPr161dme//+/ZGcnGyTolyVj0fJvy04MktERERkHclhNiwsDBs3biyzfcOGDQgLC7NJUa7KR1s6MlsscyVEREREzkHyNIOXX34Z48ePx6FDh9ClSxcAwJ9//onFixfj008/tXmBroRzZomIiIikkRxmX3jhBQQHB2PWrFn48ccfAQAtWrTA8uXL8eijj9q8QFfi71myZu+17EKZKyEiIiJyDpLCbHFxMd5//32MHDkS27dvt1dNLivcXwsASNUXIL/ICA833h6YiIiIqDKS5syqVCp89NFHKC7mnE578PN0M081OJ+RK3M1RERERLWf5A+APfTQQ9i6das9aiEAEQGeAIDz1xlmiYiIiKoiec5s79698cYbb+DIkSNo3749PD09Lfb379/fZsW5osh6Wvx1MRPJHJklIiIiqpLkMDtmzBgAwCeffFJmnyAIMBqNNa/KhXFkloiIiMh6ksOsyWSyRx10S6Q5zObJXAkRERFR7Sd5zizZV0S9kjDLaQZEREREVbM6zG7atAktW7aEXq8vsy8rKwvR0dHYtm2bTYtzRaXTDK5lFyKnkKtGEBEREVXG6jA7Z84c/POf/4ROpyuzz8fHB//6178we/Zsmxbninw81PD3dAPAebNEREREVbE6zP711194+OGHK9zfq1cv7N+/3yZFubqIeiU3T+Bas0RERESVszrMpqWlQa1WV7hfpVLh2rVrNinK1XFFAyIiIiLrWB1mGzRogKNHj1a4//DhwwgJCbFJUa6u8a0wm8wVDYiIiIgqZXWY7dOnD6ZMmYKCgoIy+/Lz8zFt2jQ88sgjNi3OVUWYw2yOzJUQERER1W5WrzP71ltvYeXKlbjnnnswbtw4NGvWDABw8uRJfPbZZzAajXjzzTftVqgrKV2e63wGR2aJiIiIKmN1mA0KCsKOHTvwwgsvYNKkSRBFEUDJXb/i4+Px2WefISgoyG6FupLSkdkbuUXIyjfAx6PiucpERERErkzSHcDCw8OxevVq3Lx5E2fPnoUoioiKioKfn5+96nNJXhoV6ntrcC27EOev5yImzFfukoiIiIhqJcm3swUAPz8/dOjQwda10B0i63mWhNkMhlkiIiKiivB2trVUREDJWrPJXJ6LiIiIqEIMs7UU15olIiIiqhrDbC0VeWtFg2SuaEBERERUoVoRZj/77DNERETA3d0dnTp1wp49eyo8dvHixRAEweLL3d3dgdU6BkdmiYiIiKome5hdvnw5EhISMG3aNBw4cAAxMTGIj49Henp6hefodDpcvXrV/JWSkuLAih2jdK3ZrHwDbuYWyVwNERERUe0ke5j95JNP8M9//hMjRoxAy5YtsXDhQmi1Wvz3v/+t8BxBEBAcHGz+qovr23q4KRGsKxlxTs7g6CwRERFReaq1NJetFBUVYf/+/Zg0aZJ5m0KhQI8ePbBz584Kz8vJyUF4eDhMJhPatWuH999/H9HR0eUeW1hYiMLCQvNjvV4PADAYDDAYDDZ6JRUrvUZ1rhVRzwOp+gL8naZH6xAvW5dWK9WkX66I/bIeeyUN+yUN+2U99koaV+2XlNcriKW38pLBlStX0KBBA+zYsQNxcXHm7a+99hq2bt2K3bt3lzln586dOHPmDNq0aYOsrCz8+9//xrZt23Ds2DE0bNiwzPHTp0/HjBkzymxfunQptFqtbV+QjS3/W4Ed6QrENzChTyOT3OUQEREROUReXh6efvppZGVlQafTVXqsrCOz1REXF2cRfLt06YIWLVrgP//5D955550yx0+aNAkJCQnmx3q9HmFhYejVq1eVzbEFg8GApKQk9OzZE2q1tNvSpv55HjvWnobROwh9+rS1U4W1S0365YrYL+uxV9KwX9KwX9Zjr6Rx1X6V/ibdGrKG2YCAACiVSqSlpVlsT0tLQ3BwsFXPoVar0bZtW5w9e7bc/RqNBhqNptzzHPlDUZ3rdWgcAOA09l3IhFKpgkIh2Ke4WsjR74+zY7+sx15Jw35Jw35Zj72SxtX6JeW1yvoBMDc3N7Rv3x4bN240bzOZTNi4caPF6GtljEYjjhw5gpCQEHuVKZvWDXygdVMiM8+AU2nZcpdDREREVOvIvppBQkICvvzyS3zzzTc4ceIEXnjhBeTm5mLEiBEAgOeee87iA2Jvv/021q9fj3PnzuHAgQN45plnkJKSgueff16ul2A3aqUC7cP9AAC7z2XIXA0RERFR7SP7nNknnngC165dw9SpU5Gamop7770Xa9euNS+3deHCBSgUtzP3zZs38c9//hOpqanw8/ND+/btsWPHDrRs2VKul2BXnRvXwx9nrmPXuRsY3jVS7nKIiIiIahXZwywAjBs3DuPGjSt335YtWywez549G7Nnz3ZAVbVD58b+AIA9529AFEUIguvMmyUiIiKqiuzTDKhyrRv4wl2twI3cIpxJz5G7HCIiIqJahWG2lnNTKRAbXjI6u4vzZomIiIgsMMw6gU6RJWF297kbMldCREREVLswzDqBzk3qAQB2J2dAxhu2EREREdU6DLNOoE1DH2hUClzPKcLf1zhvloiIiKgUw6wT0KiU5vVmd3KqAREREZEZw6yT6BR5a6oBPwRGREREZMYw6yRK15vdnXyD82aJiIiIbmGYdRIxYb5wUylwLbsQ567nyl0OERERUa3AMOsk3NVKtGvkCwDYeuqavMUQERER1RIMs06kd6sQAMDyvRc51YCIiIgIDLNOZWC7BvBQK3EqLRv7U27KXQ4RERGR7BhmnYjOXY1+MSWjs9/vviBzNURERETyY5h1MkM7hQMAfj9yFTdyi2SuhoiIiEheDLNOpk1DH7RqoENRsQk/778kdzlEREREsmKYdTKCIJhHZ5fuuQCTiR8EIyIiItfFMOuE+seEwkujQvL1XOzkHcGIiIjIhTHMOiFPjQoD2zYAAHy/O0XmaoiIiIjkwzDrpJ7u1AgAsP5YGi5n5stcDREREZE8GGadVIsQHeIa10OxScT0/x2TuxwiIiIiWTDMOrEZj0ZDpRCQdDwN64+lyl0OERERkcMxzDqxe4K88c/7GwMApv/vGHILi2WuiIiIiMixGGad3Pj/i0KYvweuZBVgzobTcpdDRERE5FAMs07Ow02Jtx9tBQD475/ncfyKXuaKiIiIiByHYbYOeLBZIPq2DoHRJOL1nw+jwGCUuyQiIiIih2CYrSOm9msJHw81jlzOwrilB1BsNMldEhEREZHdMczWEUE6d3z5XCw0KgU2nEjH5FVHIIq81S0RERHVbQyzdUjHSH/Me6otFALw475L+HjdKblLIiIiIrIrhtk6pld0MN4f2BoA8PmWv7Fgy98coSUiIqI6i2G2DnqyYyO8Gt8MAPDh2pOY+ssxzqElIiKiOolhto4a070JJvdpDkEAvtuVgue/3YfsAoPcZRERERHZFMNsHSUIAkbd3wQLhraHu1qBLaeuYfDCnbiQkSd3aUREREQ2wzBbxz3cKhjLR8WhvrcGJ1Oz8fCn27BkVwrn0RIREVGdwDDrAmLCfJE4tis6Rvojr8iItxKP4tmv9+ByZr7cpRERERHVCMOsi2jg64Fl/+yMqY+0hLtage1nryN+9jZ8ue0cior54TAiIiJyTgyzLkShEDDyvkisHt8N7Rr5IqewGO+tPoGH52zD5pPpcpdHREREJBnDrAtqXN8LK0Z3wUeD2iDAyw3nrudixOK9ePbr3dh+5jrn0xIREZHTUMldAMlDqRAwpEMYercOxrxNZ7Hoz2T8ceY6/jhzHc2CvDHyvgg8em8DuKuVcpdKREREVCGOzLo4b3c1JvdpgQ0JD2BYXDi0bkqcSsvG6z8fQcf3NuCtxCM4dDGTo7VERERUK3FklgAA4fU8MePRVkjo1QzL917ANztScDkzH0t2XcCSXRfQNNAL/WNC0ad1MJoGestdLhEREREAhlm6i4+HGqPub4Ln72uMnecy8NP+S1hz9CrOpufgk6TT+CTpNKICvdC7VTC6Nw9EmwY+UCk5wE9ERETyYJilcikUAro2DUDXpgF4+9ForD2aijVHU/HHmWs4k56DM5vOYu6ms/B2V6FrkwB0uycA3ZrWR6N6WrlLJyIiIhfCMEtV8nZXY3BsGAbHhiEr34BNJ9Ow/lga/jx7HfqCYqw9loq1x1IBAOH1tLivaQAebBaIbvcEQKPiB8iIiIjIfhhmSRIfDzUGtm2IgW0bwmgScfhSJrbfWgXhwIWbSMnIQ0rGBXy/+wK8NSr0jA5Cvzah6Ny4HjzcGGyJiIjIthhmqdqUCgFtG/mhbSM/vPhQFHIKi7Hr7wxsPX0N64+nIk1fiJUHLmPlgctQCEBkgCdahOjQMlSHtmF+uDfMlwGXiIiIaoRhlmzGS6NCj5ZB6NEyCDP6R2Nfyk38fvgK1h4rCbZ/X8vF39dy8dvhqwAAlUJAdKgO7cP9ERvhh9hwP/h5MNwSERGR9RhmyS4UCgEdI/3RMdIf0/tH41p2IY5f1eP4VT2OXs7C/pSbSNMX4q9LWfjrUhb++2cyACDMzwPBSgWy9l5Ep8b1ERXoBYVCkPnVEBERUW3FMEt2JwgCAnXuCNS5o3uzQACAKIq4dDMf+1NuYl/KDew7fxOn0rJx8WY+LkKBvf87AeAEdO4qtAsvGbVtH+7PqQlERERkgWGWZCEIAsL8tQjz12JA2wYAAH2BAfvOXcfyjfug1wTgr0tZ0BcUY8upa9hy6hqAkqkJjet7IirIG1GBXogK9MY9QV4Ir+cJNxXXuyUiInI1DLNUa+jc1egWFYDsMyb06RMLQaHEiavZJSO3KTex//xNpOoLcDotB6fTcizOVSkERAR4oml9L4TX05qDciN/LRr4ejDoEhER1VEMs1RrqZQKtG7og9YNfTCiayREUcTVrAKcSsvG2bQcnE7Lxpn0HJxNz0FOYTHO3vr+bgoBCPHxQJi/B0J9PRDi445gHw+E6NwR7OOOEB93+Hu6QRA4N5eIiMjZMMyS0xAEAaG+JYH0wVtzbwGYQ+7ptGycu5aLizfzcPFGHi7cyMPFG/nINxhxOTMflzPzK3xuN5UCwXeE22AfdwR5u6Oelxv8tG7w97z95a7mnF0iIqLagmGWnN6dIbd7M8t9oijiek4RLtzIw6WbebiSWYDUrHxczSpAqr4AVzILcD2nEEXFJly4FYCronVTwk/rVm7Q9fcsu83XQ80VGYiIiOyEYZbqNEEQUN9bg/reGrQP9yv3mKJiE9L0JeH2atbtsJueXYibuUW4cevrZl4RDEYReUVG5BVVPtJ7J4UA+JYGXK0b/DzV8PfUwP+OP3XuanhpVPByV1l8r1Zyri8REVFlakWY/eyzz/Dxxx8jNTUVMTExmDdvHjp27Fjh8StWrMCUKVNw/vx5REVF4cMPP0SfPn0cWDHVJW4qhfkDY5URRRHZhcW4mVuEjNwii6B7I68IN3JKAu+NO7brC4phEmF+LJW7WgFvdzW8b4Vbb3cV3FVKqJQCVAoFFIKIjFQFjqw7DV+tG7zd1XBXK+CmUkCjUsJNqYBGrYCbsmRbyXYF3JRKuKkUUCsF83Y3pYLzhomIyOnIHmaXL1+OhIQELFy4EJ06dcKcOXMQHx+PU6dOITAwsMzxO3bswFNPPYWZM2fikUcewdKlSzFgwAAcOHAArVq1kuEVkKsQBAE695JR1PB6nladYzCacDOvCDdzDcjILcTNXANu5BbiRq4BN/Nuh+LsAgOyC4qRXViMnIJi5BuMAIACgwkFhkJcyy6s5CoK7Eg/X/MXCJSEW6UCatUdAfjWn2qlAkqFAJVCKPlTKUCpUNx+fOtPpcXju/Yrb/0pCBCEku0KoaS3CqHke4UgQKG44/tb+8s7VqkQbj2G5fmCAIXi9rFKQYDJVIwzWQL2nL8BN7W6zPHCre8BQBBufUG49WfJY1g8FszbS4+D+bjy9996Csvjcft4y8clxwFAodGIgiIT8gzFKDSY4KlRwd/TDT4eaggALmfm40x6Ns6k5SDfYESLEB1aN/BBiI87/4FCRHWeIIqiKGcBnTp1QocOHTB//nwAgMlkQlhYGF588UW88cYbZY5/4oknkJubi99++828rXPnzrj33nuxcOHCKq+n1+vh4+ODrKws6HQ6272QChgMBqxevRp9+vSBWq22+/WcHftVwmA0IbewuCTgFhQjp7DYHHgLi40wGEUYTSIKigw4dPQEQho1Rm6RCdmFBhQaTCgsNqGo2IRCowmFBiOKjLceF5tguPV9UbEJxSZZ//pTDQkCoFYoUGQ0lbu/nqcbQnzdoTT/A0G49X1JcFcqBIt/rNw5Sq9SAOeTzyGqSRMolUrz9YDbgRvlBPCSx0K5xwsCyoRra84pb3/FPan4gIr2VPaclV3uzmsZjUYcO3YM0dHRUKkq/pBopeVXUEjlNVSyr4Izq/96pV2noic0Go04fPgwYtq0Mf9sWV7Hxu+hjXtUmYpqr8l7aDQW48DBg2jXti2USlWN6rOmnqrcf099eGrsPxYqJa/JOjJbVFSE/fv3Y9KkSeZtCoUCPXr0wM6dO8s9Z+fOnUhISLDYFh8fj8TExHKPLywsRGHh7VEtvV4PoCQ0GQyGGr6CqpVewxHXqgvYr9s81QI81WoEe1cc6g0GA4KzjqNnj8bVCv8mk1gSbo0mFBnFkpBrNMFw68+iYhMMRrFkm9EEo1GEUSwJ0sWm8v8s+b70WMBoMt21r+RPkyjCJJZM3zCJKHlsuuP7O/YZTaLFcaIIGEXR/H3p8aZynrf0XKMoIjs7B1pPz3KvYTKJEFHyuORPQETJ8+Oux3fuh/nx7fOA29tun1v2GGsJAqBVK+GuLpkeknPrHzqiCBQZTVArBTQJ8ESTQC9oVAocv6LHmWu5yLg1Jab6FNhwObkG57saJX4+f1LuIpyEEj/8fUzuIpyIEotPH5a7CADAhpfuQ3gV0/JsQUoOkDXMXr9+HUajEUFBQRbbg4KCcPJk+f9BSE1NLff41NTUco+fOXMmZsyYUWb7+vXrodXa/80olZSU5LBr1QXslzRy9Et568utOifL+rk2vZwXL9edAdi87Y5vSqZXFFucYzQBucWAwQT4agClkAkgEwDwQGPAEAFcyQVyiwWYbl3DdCtU3/lnsQkovvWn0fxYQLHp9nEWtd71jXj39nKOq2jf3aHemmtV53cJ1fn9Y7WuU40nq2iXM75Oh13fhteR+/qVjZHa6/fmNX3aP7duwTGNTUqpVF5e1asLlZJ9zqy9TZo0yWIkV6/XIywsDL169XLYNIOkpCT07NnTpX9tbi32Sxr2y3rslTTslzTsl/XYK2lctV+lv0m3hqxhNiAgAEqlEmlpaRbb09LSEBwcXO45wcHBko7XaDTQaMr+E0KtVjv0h8LR13N27Jc07Jf12Ctp2C9p2C/rsVfSuFq/pLxWWX/Z5+bmhvbt22Pjxo3mbSaTCRs3bkRcXFy558TFxVkcD5T8irWi44mIiIio7pJ9mkFCQgKGDRuG2NhYdOzYEXPmzEFubi5GjBgBAHjuuefQoEEDzJw5EwAwYcIEPPDAA5g1axb69u2LZcuWYd++ffjiiy/kfBlEREREJAPZw+wTTzyBa9euYerUqUhNTcW9996LtWvXmj/kdeHCBSgUtweQu3TpgqVLl+Ktt97C5MmTERUVhcTERK4xS0REROSCZA+zADBu3DiMGzeu3H1btmwps23w4MEYPHiwnasiIiIiotqON34nIiIiIqfFMEtERERETothloiIiIicFsMsERERETkthlkiIiIicloMs0RERETktGrF0lyOJIoiAGn3/K0Jg8GAvLw86PV6l7oNXXWxX9KwX9Zjr6Rhv6Rhv6zHXknjqv0qzWmlua0yLhdms7OzAQBhYWEyV0JERERElcnOzoaPj0+lxwiiNZG3DjGZTLhy5Qq8vb0hCILdr6fX6xEWFoaLFy9Cp9PZ/XrOjv2Shv2yHnslDfslDftlPfZKGlftlyiKyM7ORmhoqMWdYMvjciOzCoUCDRs2dPh1dTqdS/0Q1hT7JQ37ZT32Shr2Sxr2y3rslTSu2K+qRmRL8QNgREREROS0GGaJiIiIyGkxzNqZRqPBtGnToNFo5C7FKbBf0rBf1mOvpGG/pGG/rMdeScN+Vc3lPgBGRERERHUHR2aJiIiIyGkxzBIRERGR02KYJSIiIiKnxTBLRERERE6LYdbOPvvsM0RERMDd3R2dOnXCnj175C5JdjNnzkSHDh3g7e2NwMBADBgwAKdOnbI4pqCgAGPHjkW9evXg5eWFQYMGIS0tTaaKa5cPPvgAgiBg4sSJ5m3sl6XLly/jmWeeQb169eDh4YHWrVtj37595v2iKGLq1KkICQmBh4cHevTogTNnzshYsTyMRiOmTJmCyMhIeHh4oEmTJnjnnXcs7oXuyr3atm0b+vXrh9DQUAiCgMTERIv91vTmxo0bGDp0KHQ6HXx9ffGPf/wDOTk5DnwVjlNZvwwGA15//XW0bt0anp6eCA0NxXPPPYcrV65YPIer9Kuqn607jR49GoIgYM6cORbbXaVX1mCYtaPly5cjISEB06ZNw4EDBxATE4P4+Hikp6fLXZqstm7dirFjx2LXrl1ISkqCwWBAr169kJubaz7mpZdewq+//ooVK1Zg69atuHLlCh577DEZq64d9u7di//85z9o06aNxXb267abN2+ia9euUKvVWLNmDY4fP45Zs2bBz8/PfMxHH32EuXPnYuHChdi9ezc8PT0RHx+PgoICGSt3vA8//BALFizA/PnzceLECXz44Yf46KOPMG/ePPMxrtyr3NxcxMTE4LPPPit3vzW9GTp0KI4dO4akpCT89ttv2LZtG0aNGuWol+BQlfUrLy8PBw4cwJQpU3DgwAGsXLkSp06dQv/+/S2Oc5V+VfWzVWrVqlXYtWsXQkNDy+xzlV5ZRSS76dixozh27FjzY6PRKIaGhoozZ86UsaraJz09XQQgbt26VRRFUczMzBTVarW4YsUK8zEnTpwQAYg7d+6Uq0zZZWdni1FRUWJSUpL4wAMPiBMmTBBFkf262+uvvy7ed999Fe43mUxicHCw+PHHH5u3ZWZmihqNRvzhhx8cUWKt0bdvX3HkyJEW2x577DFx6NChoiiyV3cCIK5atcr82JreHD9+XAQg7t2713zMmjVrREEQxMuXLzusdjnc3a/y7NmzRwQgpqSkiKLouv2qqFeXLl0SGzRoIB49elQMDw8XZ8+ebd7nqr2qCEdm7aSoqAj79+9Hjx49zNsUCgV69OiBnTt3ylhZ7ZOVlQUA8Pf3BwDs378fBoPBonfNmzdHo0aNXLp3Y8eORd++fS36ArBfd/vf//6H2NhYDB48GIGBgWjbti2+/PJL8/7k5GSkpqZa9MvHxwedOnVyuX516dIFGzduxOnTpwEAf/31F7Zv347evXsDYK8qY01vdu7cCV9fX8TGxpqP6dGjBxQKBXbv3u3wmmubrKwsCIIAX19fAOzXnUwmE5599lm8+uqriI6OLrOfvbKkkruAuur69eswGo0ICgqy2B4UFISTJ0/KVFXtYzKZMHHiRHTt2hWtWrUCAKSmpsLNzc38H7hSQUFBSE1NlaFK+S1btgwHDhzA3r17y+xjvyydO3cOCxYsQEJCAiZPnoy9e/di/PjxcHNzw7Bhw8w9Ke/vpqv164033oBer0fz5s2hVCphNBrx3nvvYejQoQDAXlXCmt6kpqYiMDDQYr9KpYK/v7/L96+goACvv/46nnrqKeh0OgDs150+/PBDqFQqjB8/vtz97JUlhlmS1dixY3H06FFs375d7lJqrYsXL2LChAlISkqCu7u73OXUeiaTCbGxsXj//fcBAG3btsXRo0excOFCDBs2TObqapcff/wR33//PZYuXYro6GgcOnQIEydORGhoKHtFdmMwGDBkyBCIoogFCxbIXU6ts3//fnz66ac4cOAABEGQuxynwGkGdhIQEAClUlnmE+VpaWkIDg6WqaraZdy4cfjtt9+wefNmNGzY0Lw9ODgYRUVFyMzMtDjeVXu3f/9+pKeno127dlCpVFCpVNi6dSvmzp0LlUqFoKAg9usOISEhaNmypcW2Fi1a4MKFCwBg7gn/bgKvvvoq3njjDTz55JNo3bo1nn32Wbz00kuYOXMmAPaqMtb0Jjg4uMwHfouLi3Hjxg2X7V9pkE1JSUFSUpJ5VBZgv0r98ccfSE9PR6NGjcz/zU9JScHLL7+MiIgIAOzV3Rhm7cTNzQ3t27fHxo0bzdtMJhM2btyIuLg4GSuTnyiKGDduHFatWoVNmzYhMjLSYn/79u2hVqstenfq1ClcuHDBJXv30EMP4ciRIzh06JD5KzY2FkOHDjV/z37d1rVr1zJLvZ0+fRrh4eEAgMjISAQHB1v0S6/XY/fu3S7Xr7y8PCgUlv8bUCqVMJlMANiryljTm7i4OGRmZmL//v3mYzZt2gSTyYROnTo5vGa5lQbZM2fOYMOGDahXr57FfvarxLPPPovDhw9b/Dc/NDQUr776KtatWweAvSpD7k+g1WXLli0TNRqNuHjxYvH48ePiqFGjRF9fXzE1NVXu0mT1wgsviD4+PuKWLVvEq1evmr/y8vLMx4wePVps1KiRuGnTJnHfvn1iXFycGBcXJ2PVtcudqxmIIvt1pz179ogqlUp87733xDNnzojff/+9qNVqxSVLlpiP+eCDD0RfX1/xl19+EQ8fPiw++uijYmRkpJifny9j5Y43bNgwsUGDBuJvv/0mJicniytXrhQDAgLE1157zXyMK/cqOztbPHjwoHjw4EERgPjJJ5+IBw8eNH/63prePPzww2Lbtm3F3bt3i9u3bxejoqLEp556Sq6XZFeV9auoqEjs37+/2LBhQ/HQoUMW/+0vLCw0P4er9Kuqn6273b2agSi6Tq+swTBrZ/PmzRMbNWokurm5iR07dhR37dold0myA1Du16JFi8zH5Ofni2PGjBH9/PxErVYrDhw4ULx69ap8Rdcyd4dZ9svSr7/+KrZq1UrUaDRi8+bNxS+++MJiv8lkEqdMmSIGBQWJGo1GfOihh8RTp07JVK189Hq9OGHCBLFRo0aiu7u72LhxY/HNN9+0CBeu3KvNmzeX+9+qYcOGiaJoXW8yMjLEp556SvTy8hJ1Op04YsQIMTs7W4ZXY3+V9Ss5ObnC//Zv3rzZ/Byu0q+qfrbuVl6YdZVeWUMQxTtu9UJERERE5EQ4Z5aIiIiInBbDLBERERE5LYZZIiIiInJaDLNERERE5LQYZomIiIjIaTHMEhEREZHTYpglIiIiIqfFMEtERERETothlojIRQmCgMTERLnLICKqEYZZIiIZDB8+HIIglPl6+OGH5S6NiMipqOQugIjIVT388MNYtGiRxTaNRiNTNUREzokjs0REMtFoNAgODrb48vPzA1AyBWDBggXo3bs3PDw80LhxY/z0008W5x85cgT/93//Bw8PD9SrVw+jRo1CTk6OxTH//e9/ER0dDY1Gg5CQEIwbN85i//Xr1zFw4EBotVpERUXhf//7n31fNBGRjTHMEhHVUlOmTMGgQYPw119/YejQoXjyySdx4sQJAEBubi7i4+Ph5+eHvXv3YsWKFdiwYYNFWF2wYAHGjh2LUaNG4ciRI/jf//6Hpk2bWlxjxowZGDJkCA4fPow+ffpg6NChuHHjhkNfJxFRTQiiKIpyF0FE5GqGDx+OJUuWwN3d3WL75MmTMXnyZAiCgNGjR2PBggXmfZ07d0a7du3w+eef48svv8Trr7+OixcvwtPTEwCwevVq9OvXD1euXEFQUBAaNGiAESNG4N133y23BkEQ8NZbb+Gdd94BUBKQvby8sGbNGs7dJSKnwTmzREQyefDBBy3CKgD4+/ubv4+Li7PYFxcXh0OHDgEATpw4gZiYGHOQBYCuXbvCZDLh1KlTEAQBV65cwUMPPVRpDW3atDF/7+npCZ1Oh/T09Oq+JCIih2OYJSKSiaenZ5lf+9uKh4eHVcep1WqLx4IgwGQy2aMkIiK74JxZIqJaateuXWUet2jRAgDQokUL/PXXX8jNzTXv//PPP6FQKNCsWTN4e3sjIiICGzdudGjNRESOxpFZIiKZFBYWIjU11WKbSqVCQEAAAGDFihWIjY3Ffffdh++//x579uzB119/DQAYOnQopk2bhmHDhmH69Om4du0aXnzxRTz77LMICgoCAEyfPh2jR49GYGAgevfujezsbPz555948cUXHftCiYjsiGGWiEgma9euRUhIiMW2Zs2a4eTJkwBKVhpYtmwZxowZg5CQEPzwww9o2bIlAECr1WLdunWYMGECOnToAK1Wi0GDBuGTTz4xP9ewYcNQUFCA2bNn45VXXkFAQAAef/xxx71AIiIH4GoGRES1kCAIWLVqFQYMGCB3KUREtRrnzBIRERGR02KYJSIiIiKnxTmzRES1EGeAERFZhyOzREREROS0GGaJiIiIyGkxzBIRERGR02KYJSIiIiKnxTBLRERERE6LYZaIiIiInBbDLBERERE5LYZZIiIiInJa/w/HCopGB69rqAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 800x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "\n",
    "\n",
    "def train_one_epoch(net, optimizer, train_loader):\n",
    "    # Cross Entropy loss for classification when not using a softmax layer in the network\n",
    "    loss = nn.CrossEntropyLoss()\n",
    "\n",
    "    net.train()\n",
    "    avg_loss = 0\n",
    "    for data, target in train_loader:\n",
    "        optimizer.zero_grad()\n",
    "        output = net(data)\n",
    "        loss_net = loss(output, target.long())\n",
    "        loss_net.backward()\n",
    "        optimizer.step()\n",
    "        avg_loss += loss_net.item()\n",
    "\n",
    "    return avg_loss / len(train_loader)\n",
    "\n",
    "\n",
    "# Create the tiny CNN with 10 output classes\n",
    "N_EPOCHS = 150\n",
    "\n",
    "# Create a train data loader\n",
    "train_dataset = TensorDataset(torch.Tensor(x_train), torch.Tensor(y_train))\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=64)\n",
    "\n",
    "# Create a test data loader to supply batches for network evaluation (test)\n",
    "test_dataset = TensorDataset(torch.Tensor(x_test), torch.Tensor(y_test))\n",
    "test_dataloader = DataLoader(test_dataset)\n",
    "\n",
    "# Train the network with Adam, output the test set accuracy every epoch\n",
    "net = TinyCNN(10)\n",
    "losses_bits = []\n",
    "optimizer = torch.optim.Adam(net.parameters())\n",
    "for _ in tqdm(range(N_EPOCHS), desc=\"Training\"):\n",
    "    losses_bits.append(train_one_epoch(net, optimizer, train_dataloader))\n",
    "\n",
    "fig = plt.figure(figsize=(8, 4))\n",
    "plt.plot(losses_bits)\n",
    "plt.ylabel(\"Cross Entropy Loss\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.title(\"Training set loss during training\")\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "303fdc93",
   "metadata": {},
   "source": [
    "### Test the torch network in fp32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "822f1736",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy for fp32 weights and activations: 98.22%\n"
     ]
    }
   ],
   "source": [
    "def test_torch(net, test_loader):\n",
    "    \"\"\"Test the network: measure accuracy on the test set.\"\"\"\n",
    "\n",
    "    # Freeze normalization layers\n",
    "    net.eval()\n",
    "\n",
    "    all_y_pred = np.zeros((len(test_loader)), dtype=np.int64)\n",
    "    all_targets = np.zeros((len(test_loader)), dtype=np.int64)\n",
    "\n",
    "    # Iterate over the batches\n",
    "    idx = 0\n",
    "    for data, target in test_loader:\n",
    "        # Accumulate the ground truth labels\n",
    "        endidx = idx + target.shape[0]\n",
    "        all_targets[idx:endidx] = target.numpy()\n",
    "\n",
    "        # Run forward and get the predicted class id\n",
    "        output = net(data).argmax(1).detach().numpy()\n",
    "        all_y_pred[idx:endidx] = output\n",
    "\n",
    "        idx += target.shape[0]\n",
    "\n",
    "    # Print out the accuracy as a percentage\n",
    "    n_correct = np.sum(all_targets == all_y_pred)\n",
    "    print(\n",
    "        f\"Test accuracy for fp32 weights and activations: \"\n",
    "        f\"{n_correct / len(test_loader) * 100:.2f}%\"\n",
    "    )\n",
    "\n",
    "\n",
    "test_torch(net, test_dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a14bc3f",
   "metadata": {},
   "source": [
    "### Define the Concrete ML testing function\n",
    "\n",
    "We introduce the `test_with_concrete` function which allows us to test a Concrete ML model in one of two modes:\n",
    "- in FHE\n",
    "- in the clear, using simulated FHE execution\n",
    "\n",
    "Note that it is trivial to toggle between between the two modes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "978a6c4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_with_concrete(quantized_module, test_loader, use_sim):\n",
    "    \"\"\"Test a neural network that is quantized and compiled with Concrete ML.\"\"\"\n",
    "\n",
    "    # Casting the inputs into int64 is recommended\n",
    "    all_y_pred = np.zeros((len(test_loader)), dtype=np.int64)\n",
    "    all_targets = np.zeros((len(test_loader)), dtype=np.int64)\n",
    "\n",
    "    # Iterate over the test batches and accumulate predictions and ground truth labels in a vector\n",
    "    idx = 0\n",
    "    for data, target in tqdm(test_loader):\n",
    "        data = data.numpy()\n",
    "        target = target.numpy()\n",
    "\n",
    "        fhe_mode = \"simulate\" if use_sim else \"execute\"\n",
    "\n",
    "        # Quantize the inputs and cast to appropriate data type\n",
    "        y_pred = quantized_module.forward(data, fhe=fhe_mode)\n",
    "\n",
    "        endidx = idx + target.shape[0]\n",
    "\n",
    "        # Accumulate the ground truth labels\n",
    "        all_targets[idx:endidx] = target\n",
    "\n",
    "        # Get the predicted class id and accumulate the predictions\n",
    "        y_pred = np.argmax(y_pred, axis=1)\n",
    "        all_y_pred[idx:endidx] = y_pred\n",
    "\n",
    "        # Update the index\n",
    "        idx += target.shape[0]\n",
    "\n",
    "    # Compute and report results\n",
    "    n_correct = np.sum(all_targets == all_y_pred)\n",
    "\n",
    "    return n_correct / len(test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cccf0b5",
   "metadata": {},
   "source": [
    "### Test the network using Simulation\n",
    "\n",
    "Note that this is not a test in FHE. The simulated FHE mode gives \n",
    "insight about the impact of FHE execution on the accuracy.\n",
    "\n",
    "The torch neural network is converted to FHE by Concrete ML using a dedicated function, `compile_torch_model`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "33250d57",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/450 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 1/450 [00:00<03:13,  2.32it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  1%|          | 3/450 [00:00<01:14,  6.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  1%|▏         | 6/450 [00:00<00:38, 11.64it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  2%|▏         | 9/450 [00:00<00:27, 16.03it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 12/450 [00:00<00:22, 19.30it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 15/450 [00:01<00:19, 21.77it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  4%|▍         | 18/450 [00:01<00:18, 23.58it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|▍         | 21/450 [00:01<00:17, 24.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|▌         | 24/450 [00:01<00:16, 25.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  6%|▌         | 27/450 [00:01<00:15, 26.54it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 30/450 [00:01<00:15, 27.01it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 33/450 [00:01<00:15, 27.34it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 36/450 [00:01<00:15, 27.57it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  9%|▊         | 39/450 [00:01<00:14, 27.72it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  9%|▉         | 42/450 [00:01<00:14, 27.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 45/450 [00:02<00:14, 27.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 11%|█         | 48/450 [00:02<00:14, 28.01it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 11%|█▏        | 51/450 [00:02<00:14, 28.07it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 12%|█▏        | 54/450 [00:02<00:14, 28.12it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 57/450 [00:02<00:13, 28.13it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 60/450 [00:02<00:13, 28.12it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 14%|█▍        | 63/450 [00:02<00:13, 28.09it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▍        | 66/450 [00:02<00:13, 28.10it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 69/450 [00:02<00:13, 28.12it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 16%|█▌        | 72/450 [00:03<00:13, 28.15it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 75/450 [00:03<00:13, 28.17it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 78/450 [00:03<00:13, 28.16it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 18%|█▊        | 81/450 [00:03<00:13, 28.21it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 19%|█▊        | 84/450 [00:03<00:12, 28.17it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 19%|█▉        | 87/450 [00:03<00:13, 27.82it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 90/450 [00:03<00:12, 27.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 21%|██        | 93/450 [00:03<00:12, 28.01it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 21%|██▏       | 96/450 [00:03<00:12, 28.05it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 22%|██▏       | 99/450 [00:03<00:12, 28.11it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 102/450 [00:04<00:12, 28.14it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 105/450 [00:04<00:12, 28.10it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 24%|██▍       | 108/450 [00:04<00:12, 28.13it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▍       | 111/450 [00:04<00:12, 28.20it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 114/450 [00:04<00:11, 28.21it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 26%|██▌       | 117/450 [00:04<00:11, 28.25it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 120/450 [00:04<00:11, 28.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 123/450 [00:04<00:11, 28.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 28%|██▊       | 126/450 [00:04<00:11, 28.45it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 29%|██▊       | 129/450 [00:05<00:11, 28.57it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 29%|██▉       | 132/450 [00:05<00:11, 28.56it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 135/450 [00:05<00:11, 28.54it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 31%|███       | 138/450 [00:05<00:10, 28.65it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 31%|███▏      | 141/450 [00:05<00:10, 28.70it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 32%|███▏      | 144/450 [00:05<00:10, 28.66it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 147/450 [00:05<00:10, 28.75it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 150/450 [00:05<00:10, 28.85it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 34%|███▍      | 153/450 [00:05<00:10, 28.90it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▍      | 156/450 [00:05<00:10, 28.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▌      | 159/450 [00:06<00:10, 28.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 36%|███▌      | 162/450 [00:06<00:09, 28.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 165/450 [00:06<00:09, 28.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 168/450 [00:06<00:09, 28.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 38%|███▊      | 171/450 [00:06<00:09, 28.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 39%|███▊      | 174/450 [00:06<00:09, 28.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 39%|███▉      | 177/450 [00:06<00:09, 28.94it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 180/450 [00:06<00:09, 28.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 41%|████      | 183/450 [00:06<00:09, 28.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 41%|████▏     | 186/450 [00:07<00:09, 28.84it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 42%|████▏     | 189/450 [00:07<00:09, 28.85it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|████▎     | 192/450 [00:07<00:08, 28.88it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|████▎     | 195/450 [00:07<00:08, 28.91it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████▍     | 198/450 [00:07<00:08, 28.92it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▍     | 201/450 [00:07<00:08, 28.93it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▌     | 204/450 [00:07<00:08, 28.95it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 46%|████▌     | 207/450 [00:07<00:08, 28.96it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|████▋     | 210/450 [00:07<00:08, 28.98it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|████▋     | 213/450 [00:07<00:08, 28.99it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 48%|████▊     | 216/450 [00:08<00:08, 28.98it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 49%|████▊     | 219/450 [00:08<00:07, 29.04it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 49%|████▉     | 222/450 [00:08<00:07, 29.16it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 225/450 [00:08<00:07, 29.24it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 51%|█████     | 228/450 [00:08<00:07, 29.29it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 51%|█████▏    | 231/450 [00:08<00:07, 29.33it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 52%|█████▏    | 234/450 [00:08<00:07, 29.35it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 53%|█████▎    | 237/450 [00:08<00:07, 29.38it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 53%|█████▎    | 240/450 [00:08<00:07, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 54%|█████▍    | 243/450 [00:08<00:07, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▍    | 246/450 [00:09<00:06, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▌    | 249/450 [00:09<00:06, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 56%|█████▌    | 252/450 [00:09<00:06, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|█████▋    | 255/450 [00:09<00:06, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|█████▋    | 258/450 [00:09<00:06, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 58%|█████▊    | 261/450 [00:09<00:06, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 59%|█████▊    | 264/450 [00:09<00:06, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 59%|█████▉    | 267/450 [00:09<00:06, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 270/450 [00:09<00:06, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 61%|██████    | 273/450 [00:09<00:06, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 61%|██████▏   | 276/450 [00:10<00:05, 29.39it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 62%|██████▏   | 279/450 [00:10<00:05, 29.38it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 63%|██████▎   | 282/450 [00:10<00:05, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 63%|██████▎   | 285/450 [00:10<00:05, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 64%|██████▍   | 288/450 [00:10<00:05, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▍   | 291/450 [00:10<00:05, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 294/450 [00:10<00:05, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 66%|██████▌   | 297/450 [00:10<00:05, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████▋   | 300/450 [00:10<00:05, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████▋   | 303/450 [00:11<00:04, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 68%|██████▊   | 306/450 [00:11<00:04, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 69%|██████▊   | 309/450 [00:11<00:04, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 69%|██████▉   | 312/450 [00:11<00:04, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 315/450 [00:11<00:04, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 71%|███████   | 318/450 [00:11<00:04, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 71%|███████▏  | 321/450 [00:11<00:04, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 72%|███████▏  | 324/450 [00:11<00:04, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 73%|███████▎  | 327/450 [00:11<00:04, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 73%|███████▎  | 330/450 [00:11<00:04, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 74%|███████▍  | 333/450 [00:12<00:03, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▍  | 336/450 [00:12<00:03, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▌  | 339/450 [00:12<00:03, 29.39it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▌  | 342/450 [00:12<00:03, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 77%|███████▋  | 345/450 [00:12<00:03, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 77%|███████▋  | 348/450 [00:12<00:03, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████▊  | 351/450 [00:12<00:03, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 79%|███████▊  | 354/450 [00:12<00:03, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 79%|███████▉  | 357/450 [00:12<00:03, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 360/450 [00:12<00:03, 29.45it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 81%|████████  | 363/450 [00:13<00:02, 29.45it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 81%|████████▏ | 366/450 [00:13<00:02, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 82%|████████▏ | 369/450 [00:13<00:02, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 83%|████████▎ | 372/450 [00:13<00:02, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 83%|████████▎ | 375/450 [00:13<00:02, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 84%|████████▍ | 378/450 [00:13<00:02, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▍ | 381/450 [00:13<00:02, 28.83it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 384/450 [00:13<00:02, 29.00it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|████████▌ | 387/450 [00:13<00:02, 29.14it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 87%|████████▋ | 390/450 [00:13<00:02, 29.23it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 87%|████████▋ | 393/450 [00:14<00:01, 29.29it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 88%|████████▊ | 396/450 [00:14<00:01, 29.34it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 89%|████████▊ | 399/450 [00:14<00:01, 29.34it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 89%|████████▉ | 402/450 [00:14<00:01, 29.37it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 405/450 [00:14<00:01, 29.39it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 91%|█████████ | 408/450 [00:14<00:01, 29.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 91%|█████████▏| 411/450 [00:14<00:01, 29.41it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 92%|█████████▏| 414/450 [00:14<00:01, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 93%|█████████▎| 417/450 [00:14<00:01, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 93%|█████████▎| 420/450 [00:15<00:01, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 94%|█████████▍| 423/450 [00:15<00:00, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▍| 426/450 [00:15<00:00, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▌| 429/450 [00:15<00:00, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 96%|█████████▌| 432/450 [00:15<00:00, 29.42it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 97%|█████████▋| 435/450 [00:15<00:00, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 97%|█████████▋| 438/450 [00:15<00:00, 29.43it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 98%|█████████▊| 441/450 [00:15<00:00, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 99%|█████████▊| 444/450 [00:15<00:00, 29.44it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 99%|█████████▉| 447/450 [00:15<00:00, 29.45it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 450/450 [00:16<00:00, 29.45it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 450/450 [00:16<00:00, 28.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simulated FHE execution for 6 bit network accuracy: 0.97%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "n_bits = 6\n",
    "\n",
    "use_gpu_if_available = False\n",
    "device = \"cuda\" if use_gpu_if_available and check_gpu_available() else \"cpu\"\n",
    "\n",
    "q_module = compile_torch_model(net, x_train, rounding_threshold_bits=6, p_error=0.1, device=device)\n",
    "\n",
    "start_time = time.time()\n",
    "accs = test_with_concrete(\n",
    "    q_module,\n",
    "    test_dataloader,\n",
    "    use_sim=True,\n",
    ")\n",
    "sim_time = time.time() - start_time\n",
    "\n",
    "print(f\"Simulated FHE execution for {n_bits} bit network accuracy: {accs:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2875e825",
   "metadata": {},
   "source": [
    "### Generate Keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6e8b6471",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keygen time: 1.88s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Generate keys first\n",
    "t = time.time()\n",
    "q_module.fhe_circuit.keygen()\n",
    "print(f\"Keygen time: {time.time()-t:.2f}s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4dd257f",
   "metadata": {},
   "source": [
    "### 3. Execute in FHE on encrypted data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5a82392b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/100 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  1%|          | 1/100 [00:02<04:29,  2.73s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  2%|▏         | 2/100 [00:04<03:55,  2.41s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 3/100 [00:14<08:51,  5.48s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  4%|▍         | 4/100 [00:18<07:51,  4.91s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|▌         | 5/100 [00:20<06:14,  3.95s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  6%|▌         | 6/100 [00:22<05:19,  3.40s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 7/100 [00:24<04:40,  3.02s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 8/100 [00:27<04:14,  2.76s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  9%|▉         | 9/100 [00:31<05:07,  3.38s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 10/100 [00:36<05:34,  3.72s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 11%|█         | 11/100 [00:38<04:53,  3.29s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 12%|█▏        | 12/100 [00:41<04:25,  3.02s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 13/100 [00:43<04:03,  2.80s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 14%|█▍        | 14/100 [00:45<03:47,  2.65s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 15/100 [00:47<03:35,  2.53s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 16%|█▌        | 16/100 [00:50<03:25,  2.45s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 17/100 [00:52<03:18,  2.40s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 18%|█▊        | 18/100 [00:54<03:13,  2.36s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 19%|█▉        | 19/100 [00:58<03:47,  2.81s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 20/100 [01:00<03:35,  2.69s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 21%|██        | 21/100 [01:04<03:57,  3.00s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 22%|██▏       | 22/100 [01:08<04:12,  3.24s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 23/100 [01:10<03:47,  2.95s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 24%|██▍       | 24/100 [01:13<03:28,  2.75s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 25/100 [01:15<03:17,  2.63s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 26%|██▌       | 26/100 [01:17<03:06,  2.52s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 27/100 [01:19<02:58,  2.44s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 28%|██▊       | 28/100 [01:22<02:51,  2.38s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 29%|██▉       | 29/100 [01:24<02:46,  2.35s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 30/100 [01:26<02:42,  2.32s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 31%|███       | 31/100 [01:28<02:38,  2.30s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 32%|███▏      | 32/100 [01:32<02:53,  2.56s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 33/100 [01:34<02:45,  2.47s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 34%|███▍      | 34/100 [01:36<02:39,  2.41s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▌      | 35/100 [01:41<03:25,  3.16s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 36%|███▌      | 36/100 [01:43<03:05,  2.89s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 37/100 [01:46<02:49,  2.70s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 38%|███▊      | 38/100 [01:48<02:38,  2.56s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 39%|███▉      | 39/100 [01:52<02:57,  2.91s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 40/100 [01:54<02:43,  2.73s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 41%|████      | 41/100 [01:56<02:32,  2.59s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 42%|████▏     | 42/100 [01:58<02:24,  2.49s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|████▎     | 43/100 [02:01<02:17,  2.42s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████▍     | 44/100 [02:04<02:26,  2.62s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▌     | 45/100 [02:06<02:18,  2.51s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 46%|████▌     | 46/100 [02:08<02:11,  2.44s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|████▋     | 47/100 [02:10<02:06,  2.38s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 48%|████▊     | 48/100 [02:14<02:15,  2.61s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 49%|████▉     | 49/100 [02:16<02:12,  2.61s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 50/100 [02:19<02:09,  2.59s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 51%|█████     | 51/100 [02:22<02:09,  2.64s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 52%|█████▏    | 52/100 [02:24<02:00,  2.52s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 53%|█████▎    | 53/100 [02:26<01:54,  2.43s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 54%|█████▍    | 54/100 [02:28<01:49,  2.38s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▌    | 55/100 [02:30<01:45,  2.34s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 56%|█████▌    | 56/100 [02:33<01:41,  2.31s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 57%|█████▋    | 57/100 [02:35<01:39,  2.32s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 58%|█████▊    | 58/100 [02:37<01:36,  2.30s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 59%|█████▉    | 59/100 [02:40<01:33,  2.29s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 60/100 [02:42<01:31,  2.29s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 61%|██████    | 61/100 [02:44<01:29,  2.29s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 62%|██████▏   | 62/100 [02:46<01:26,  2.28s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 63%|██████▎   | 63/100 [02:49<01:24,  2.27s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 64%|██████▍   | 64/100 [02:51<01:21,  2.27s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 65/100 [02:53<01:19,  2.26s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 66%|██████▌   | 66/100 [02:55<01:16,  2.25s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████▋   | 67/100 [02:58<01:14,  2.25s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 68%|██████▊   | 68/100 [03:00<01:12,  2.26s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 69%|██████▉   | 69/100 [03:02<01:10,  2.26s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 70/100 [03:05<01:08,  2.28s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 71%|███████   | 71/100 [03:07<01:05,  2.27s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 72%|███████▏  | 72/100 [03:09<01:03,  2.27s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 73%|███████▎  | 73/100 [03:12<01:10,  2.61s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 74%|███████▍  | 74/100 [03:15<01:05,  2.53s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▌  | 75/100 [03:17<01:01,  2.45s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|███████▌  | 76/100 [03:19<00:57,  2.39s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 77%|███████▋  | 77/100 [03:22<00:54,  2.35s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████▊  | 78/100 [03:24<00:51,  2.34s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 79%|███████▉  | 79/100 [03:26<00:48,  2.32s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 80/100 [03:28<00:45,  2.30s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 81%|████████  | 81/100 [03:31<00:43,  2.28s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 82%|████████▏ | 82/100 [03:33<00:43,  2.44s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 83%|████████▎ | 83/100 [03:36<00:40,  2.40s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 84%|████████▍ | 84/100 [03:38<00:38,  2.42s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 85/100 [03:40<00:35,  2.38s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 86%|████████▌ | 86/100 [03:44<00:35,  2.57s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 87%|████████▋ | 87/100 [03:46<00:32,  2.48s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 88%|████████▊ | 88/100 [03:48<00:28,  2.41s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 89%|████████▉ | 89/100 [03:50<00:25,  2.36s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 90/100 [03:53<00:23,  2.38s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 91%|█████████ | 91/100 [03:55<00:21,  2.34s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 92%|█████████▏| 92/100 [03:57<00:18,  2.31s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 93%|█████████▎| 93/100 [03:59<00:16,  2.30s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 94%|█████████▍| 94/100 [04:02<00:13,  2.29s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▌| 95/100 [04:04<00:11,  2.28s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 96%|█████████▌| 96/100 [04:06<00:09,  2.27s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 97%|█████████▋| 97/100 [04:08<00:06,  2.27s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 98%|█████████▊| 98/100 [04:11<00:04,  2.26s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 99%|█████████▉| 99/100 [04:13<00:02,  2.26s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 100/100 [04:15<00:00,  2.26s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100%|██████████| 100/100 [04:15<00:00,  2.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time per inference in FHE: 2.56 with 99.00% accuracy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Run inference in FHE on a single encrypted example\n",
    "mini_test_dataset = TensorDataset(torch.Tensor(x_test[:100, :]), torch.Tensor(y_test[:100]))\n",
    "mini_test_dataloader = DataLoader(mini_test_dataset)\n",
    "\n",
    "t = time.time()\n",
    "accuracy_test = test_with_concrete(\n",
    "    q_module,\n",
    "    mini_test_dataloader,\n",
    "    use_sim=False,\n",
    ")\n",
    "elapsed_time = time.time() - t\n",
    "time_per_inference = elapsed_time / len(mini_test_dataset)\n",
    "accuracy_percentage = 100 * accuracy_test\n",
    "\n",
    "print(\n",
    "    f\"Time per inference in FHE: {time_per_inference:.2f} \"\n",
    "    f\"with {accuracy_percentage:.2f}% accuracy\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "550f48bf",
   "metadata": {},
   "source": [
    "### Conclusion\n",
    "\n",
    "In this example, a simple CNN model is trained with torch and reach 99% accuracy in clear. The model is then converted to FHE and evaluated over 100 samples in FHE.\n",
    "\n",
    "The model in FHE achieves **the same accuracy** as the original torch model with a FHE execution time of **2.9 seconds** per image."
   ]
  }
 ],
 "metadata": {
  "execution": {
   "timeout": 10800
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
